From b91cda365ac1c68318faa8567218e0ab98add18a Mon Sep 17 00:00:00 2001 From: suoxiaocong Date: Thu, 15 Jun 2023 01:08:24 +0800 Subject: [PATCH 01/38] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E6=A1=86?= =?UTF-8?q?=E6=9E=B6=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/nkd/deploy.go | 21 +++++++++ cmd/nkd/generate.go | 20 +++++++++ cmd/nkd/join.go | 21 +++++++++ cmd/nkd/main.go | 17 +++++++ cmd/nkd/root.go | 21 +++++++++ cmd/nkd/upgrade.go | 21 +++++++++ go.mod | 14 ++++++ go.sum | 14 ++++++ housekeeper/daemon/daemon.go | 0 housekeeper/operator/operator.go | 0 pkg/apimonitor.go | 19 ++++++++ pkg/bootconfig/ignitioncreator.go | 10 +++++ pkg/bootconfig/initconfig.go | 3 ++ pkg/cert/etcd.go | 17 +++++++ pkg/deployer/clusterInfo.go | 40 +++++++++++++++++ pkg/deployer/phase/base.go | 5 +++ pkg/deployer/phase/generate.go | 21 +++++++++ pkg/deployer/phase/init.go | 15 +++++++ pkg/deployer/phase/join.go | 1 + pkg/deployer/playbook/base.go | 28 ++++++++++++ pkg/deployer/playbook/oneshot.go | 15 +++++++ pkg/deployer/playbook/upgrade.go | 6 +++ pkg/deployer/state.go | 5 +++ pkg/deployer/template.go | 10 +++++ pkg/deployer/utils.go | 17 +++++++ pkg/deployer/workflow.go | 74 +++++++++++++++++++++++++++++++ pkg/infra/assets.go | 16 +++++++ pkg/infra/factory.go | 18 ++++++++ pkg/infra/infradeployer.go | 17 +++++++ pkg/infra/initconfig.go | 10 +++++ pkg/infra/openstack.go | 10 +++++ pkg/manager/manager.go | 0 pkg/osimage/osimage.go | 0 pkg/registry/registry.go | 0 34 files changed, 506 insertions(+) create mode 100755 cmd/nkd/deploy.go create mode 100755 cmd/nkd/generate.go create mode 100755 cmd/nkd/join.go create mode 100755 cmd/nkd/main.go create mode 100755 cmd/nkd/root.go create mode 100755 cmd/nkd/upgrade.go create mode 100755 go.mod create mode 100755 go.sum create mode 100644 housekeeper/daemon/daemon.go create mode 100644 housekeeper/operator/operator.go create mode 100755 pkg/apimonitor.go create mode 100755 pkg/bootconfig/ignitioncreator.go create mode 100755 pkg/bootconfig/initconfig.go create mode 100755 pkg/cert/etcd.go create mode 100755 pkg/deployer/clusterInfo.go create mode 100755 pkg/deployer/phase/base.go create mode 100755 pkg/deployer/phase/generate.go create mode 100755 pkg/deployer/phase/init.go create mode 100755 pkg/deployer/phase/join.go create mode 100755 pkg/deployer/playbook/base.go create mode 100755 pkg/deployer/playbook/oneshot.go create mode 100755 pkg/deployer/playbook/upgrade.go create mode 100755 pkg/deployer/state.go create mode 100755 pkg/deployer/template.go create mode 100755 pkg/deployer/utils.go create mode 100755 pkg/deployer/workflow.go create mode 100755 pkg/infra/assets.go create mode 100755 pkg/infra/factory.go create mode 100755 pkg/infra/infradeployer.go create mode 100755 pkg/infra/initconfig.go create mode 100755 pkg/infra/openstack.go create mode 100644 pkg/manager/manager.go create mode 100644 pkg/osimage/osimage.go create mode 100644 pkg/registry/registry.go diff --git a/cmd/nkd/deploy.go b/cmd/nkd/deploy.go new file mode 100755 index 0000000..0caebe6 --- /dev/null +++ b/cmd/nkd/deploy.go @@ -0,0 +1,21 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var ( + deployCommand = &cobra.Command{ + Use: "deploy [nodes]", + Short: "deploy masters ", + Long: "deploy cluster bootconfig for a new cluster", + RunE: deploy, + Args: cobra.MaximumNArgs(1), + } +) + + +func deploy(command *cobra.Command, args []string) error{ + println("deploy") + return nil +} diff --git a/cmd/nkd/generate.go b/cmd/nkd/generate.go new file mode 100755 index 0000000..961909c --- /dev/null +++ b/cmd/nkd/generate.go @@ -0,0 +1,20 @@ +package main + +import "github.com/spf13/cobra" + +var ( + generateCommand = &cobra.Command{ + Use: "generate [options] [REGISTRY]", + Short: "generate cluster bootconfig", + Long: "generate cluster bootconfig for a new cluster", + RunE: generate, + Args: cobra.MaximumNArgs(1), + } +) + + +func generate(command *cobra.Command, args []string) error{ + println("generate") + return nil +} + diff --git a/cmd/nkd/join.go b/cmd/nkd/join.go new file mode 100755 index 0000000..1a7abf9 --- /dev/null +++ b/cmd/nkd/join.go @@ -0,0 +1,21 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var ( + joinCommand = &cobra.Command{ + Use: "join [nodes]", + Short: "join nodes ", + Long: "join nodes to cluster", + RunE: join, + Args: cobra.MaximumNArgs(1), + } +) + + +func join(command *cobra.Command, args []string) error{ + println("join") + return nil +} diff --git a/cmd/nkd/main.go b/cmd/nkd/main.go new file mode 100755 index 0000000..66ce32f --- /dev/null +++ b/cmd/nkd/main.go @@ -0,0 +1,17 @@ +package main + +import "github.com/spf13/cobra" + +func addCommands(command *cobra.Command){ + command.AddCommand(generateCommand) + command.AddCommand(deployCommand) + //command.AddCommand(initCommand) + command.AddCommand(joinCommand) + command.AddCommand(upgradeCommand) +} + + +func main() { + addCommands(rootCmd) + _ = rootCmd.Execute() +} diff --git a/cmd/nkd/root.go b/cmd/nkd/root.go new file mode 100755 index 0000000..d8a4995 --- /dev/null +++ b/cmd/nkd/root.go @@ -0,0 +1,21 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "nkd [options]", + Long: "deploy k8s clusters", + SilenceUsage: true, + SilenceErrors: true, + TraverseChildren: true, + //PersistentPreRunE: persistentPreRunE, + //RunE: validate.SubCommandExists, + //PersistentPostRunE: persistentPostRunE, + //Version: version.Version.String(), + DisableFlagsInUseLine: true, + } + +) diff --git a/cmd/nkd/upgrade.go b/cmd/nkd/upgrade.go new file mode 100755 index 0000000..087630f --- /dev/null +++ b/cmd/nkd/upgrade.go @@ -0,0 +1,21 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var ( + upgradeCommand = &cobra.Command{ + Use: "deploy [nodes]", + Short: "deploy masters ", + Long: "deploy cluster bootconfig for a new cluster", + RunE: upgrade, + Args: cobra.MaximumNArgs(1), + } +) + + +func upgrade(command *cobra.Command, args []string) error{ + println("deploy") + return nil +} diff --git a/go.mod b/go.mod new file mode 100755 index 0000000..0fe9ed1 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module gitee.com/openeuler/nestos-kubernetes-deployer + +go 1.18 + +require ( + github.com/lithammer/dedent v1.1.0 + github.com/pkg/errors v0.9.1 + github.com/spf13/cobra v1.7.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/go.sum b/go.sum new file mode 100755 index 0000000..1516f81 --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/housekeeper/daemon/daemon.go b/housekeeper/daemon/daemon.go new file mode 100644 index 0000000..e69de29 diff --git a/housekeeper/operator/operator.go b/housekeeper/operator/operator.go new file mode 100644 index 0000000..e69de29 diff --git a/pkg/apimonitor.go b/pkg/apimonitor.go new file mode 100755 index 0000000..39d5824 --- /dev/null +++ b/pkg/apimonitor.go @@ -0,0 +1,19 @@ +package pkg + +import "time" + +type ApiMonitor struct { + Endpoint string +} + +func (l ApiMonitor) WaitForClusterReady(timeout time.Duration) { + return +} + +func (l ApiMonitor) WaitForMastersReady(timeout time.Duration){ +} + +func (l ApiMonitor) WaitForWorkersReady(timeout time.Duration){ + return +} + diff --git a/pkg/bootconfig/ignitioncreator.go b/pkg/bootconfig/ignitioncreator.go new file mode 100755 index 0000000..25a8517 --- /dev/null +++ b/pkg/bootconfig/ignitioncreator.go @@ -0,0 +1,10 @@ +package bootconfig + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra" + +type IgnitionAssembler struct { +} + +func (i IgnitionAssembler) Assemble(assets infra.Assets) infra.InitConfig { + return infra.InitConfig{} +} diff --git a/pkg/bootconfig/initconfig.go b/pkg/bootconfig/initconfig.go new file mode 100755 index 0000000..b1be118 --- /dev/null +++ b/pkg/bootconfig/initconfig.go @@ -0,0 +1,3 @@ +package bootconfig + + diff --git a/pkg/cert/etcd.go b/pkg/cert/etcd.go new file mode 100755 index 0000000..93cc24f --- /dev/null +++ b/pkg/cert/etcd.go @@ -0,0 +1,17 @@ +package cert + +import ( + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra" +) + + + +type EtcdCaGenerator struct { +} + +func (e EtcdCaGenerator) GenerateAssets() infra.Assets { + return nil +} + +func init() { +} \ No newline at end of file diff --git a/pkg/deployer/clusterInfo.go b/pkg/deployer/clusterInfo.go new file mode 100755 index 0000000..3964b25 --- /dev/null +++ b/pkg/deployer/clusterInfo.go @@ -0,0 +1,40 @@ +package deployer + + +type ClusterInfo struct { + Name string `yaml:"name"` + ApiEndpoint string `yaml:"api_endpoint"` + Kubernetes Kubernetes `yaml:"kubernetes"` + ClusterCIDR string `yaml:"cluster_cidr"` + PodCIDR string `yaml:"pod_cidr"` + ServiceCIDR string `yaml:"service_cidr"` + InfraDriver string `yaml:"infra_driver"` + OsType string `yaml:"os_type"` + OsImage string `yaml:"os_image"` + OsConfigList []OsConfig `yaml:"os_config_list"` + EtcdType string `yaml:"etcd_type"` +} + +type Kubernetes struct { + KubernetesVersion string `yaml:"kubernetes_version"` +} + +type OsConfig struct { + Roles []string `yaml:"roles"` + DiskSize string `yaml:"disk_size"` + Image string `yaml:"image"` + MemorySize string `yaml:"memory_size"` + Cpus int `yaml:"cpus"` +} + + + + +func ParseFromFile(filename string) ClusterInfo{ + return ClusterInfo{} +} + + +func (* ClusterInfo)GenerateToFile(filename string) error{ + return nil +} \ No newline at end of file diff --git a/pkg/deployer/phase/base.go b/pkg/deployer/phase/base.go new file mode 100755 index 0000000..67d3ce8 --- /dev/null +++ b/pkg/deployer/phase/base.go @@ -0,0 +1,5 @@ +package phase + +type Phase interface { + Do() +} \ No newline at end of file diff --git a/pkg/deployer/phase/generate.go b/pkg/deployer/phase/generate.go new file mode 100755 index 0000000..4d7161e --- /dev/null +++ b/pkg/deployer/phase/generate.go @@ -0,0 +1,21 @@ +package phase + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra" + + +type GeneratePhase struct{ + Mode string // master, node +} + +func (p GeneratePhase)GenerateAssets() infra.Assets{ + //managers = [ + //"etcd", + //"manifests", + //"certs", + //] + //assets = dict() + //for i in managers: + // assets.merge(i.generateAssets()) + //return assets + return infra.Assets{} +} diff --git a/pkg/deployer/phase/init.go b/pkg/deployer/phase/init.go new file mode 100755 index 0000000..b0e860c --- /dev/null +++ b/pkg/deployer/phase/init.go @@ -0,0 +1,15 @@ +package phase + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra" + +type InitPhase struct { +} + +func (p InitPhase) Do() { + generator := infra.GetBootConfigAssembler(clusterInfo.OsType) + masterInitConfig := generator.Assemble(generateInitAssets()) + masterSpec := parseSpecFromClusterInfo(clusterInfo) + _ = infraDeployer.Create(masterSpec, masterInitConfig) + apiMonitor.WaitForMastersReady(0) + +} \ No newline at end of file diff --git a/pkg/deployer/phase/join.go b/pkg/deployer/phase/join.go new file mode 100755 index 0000000..367bfa5 --- /dev/null +++ b/pkg/deployer/phase/join.go @@ -0,0 +1 @@ +package phase diff --git a/pkg/deployer/playbook/base.go b/pkg/deployer/playbook/base.go new file mode 100755 index 0000000..c945088 --- /dev/null +++ b/pkg/deployer/playbook/base.go @@ -0,0 +1,28 @@ +package playbook + +import ( + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/deployer" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/deployer/phase" +) + +type Playbook struct { + clusterInfo deployer.ClusterInfo + phases []phase.Phase +} + + +func (p Playbook) Start() { + for _, i := range p.phases{ + p.Run(i) + } +} + + +func (p *Playbook) AddPhase(phase phase.Phase){ + p.phases = append(p.phases, phase) +} + + +func (p Playbook) Run(phase phase.Phase) { + +} \ No newline at end of file diff --git a/pkg/deployer/playbook/oneshot.go b/pkg/deployer/playbook/oneshot.go new file mode 100755 index 0000000..d0a498c --- /dev/null +++ b/pkg/deployer/playbook/oneshot.go @@ -0,0 +1,15 @@ +package playbook + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/deployer/phase" + +func OneshotPlaybook() Playbook{ + return Playbook{ + phase.InitPhase{}, + } +} + +//managers = [ +//"etcd", +//"manifests", +//"certs", +//] \ No newline at end of file diff --git a/pkg/deployer/playbook/upgrade.go b/pkg/deployer/playbook/upgrade.go new file mode 100755 index 0000000..db1daaf --- /dev/null +++ b/pkg/deployer/playbook/upgrade.go @@ -0,0 +1,6 @@ +package playbook + +type UpgradePlabook struct { + +} + diff --git a/pkg/deployer/state.go b/pkg/deployer/state.go new file mode 100755 index 0000000..9b7a6f9 --- /dev/null +++ b/pkg/deployer/state.go @@ -0,0 +1,5 @@ +package deployer + +const ( + state_ready = iota +) diff --git a/pkg/deployer/template.go b/pkg/deployer/template.go new file mode 100755 index 0000000..c1d01d1 --- /dev/null +++ b/pkg/deployer/template.go @@ -0,0 +1,10 @@ +package deployer + +import ( + "github.com/pkg/errors" + "strings" + "text/template" + + "github.com/lithammer/dedent" +) + diff --git a/pkg/deployer/utils.go b/pkg/deployer/utils.go new file mode 100755 index 0000000..fc18c22 --- /dev/null +++ b/pkg/deployer/utils.go @@ -0,0 +1,17 @@ +package deployer + +// 通过grpc请求handler完成操作 + +func DownloadFile(content, path string){ + +} + + +func PullImage(){ + +} + + +func WriteConfig(content, path string){ + +} diff --git a/pkg/deployer/workflow.go b/pkg/deployer/workflow.go new file mode 100755 index 0000000..e4e5f4b --- /dev/null +++ b/pkg/deployer/workflow.go @@ -0,0 +1,74 @@ +package deployer + +import ( + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/deployer/phase" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra" +) +// TODO 断点续传机制 + +func InitCluster(filename string) error { + // TODO cluster和vm的状态机制, init, pending, + // cluster state: pending, running, stopped + // infra state: none, pending, created, starting, running, stopping, stopped + // k8s state: TODO + + + clusterInfo := ParseFromFile(filename) + infraDeployer := infra.GetInfraDeployer(clusterInfo.InfraDriver) + bootConfigAssembler := infra.GetBootConfigAssembler(clusterInfo.InfraDriver) + + infraSpec := parseSpecFromClusterInfo(clusterInfo) + assets := generateInitAssets() + bootConfig := bootConfigAssembler.Assemble(assets) + _ = infraDeployer.Create(infraSpec, bootConfig) + apiMonitor := pkg.ApiMonitor{clusterInfo.ApiEndpoint} + apiMonitor.WaitForMastersReady(0) + return nil +} + + + +func JoinToCluster(filename string) { + clusterInfo := ParseFromFile(filename) + infraDeployer := infra.GetInfraDeployer(clusterInfo.InfraDriver) + apiMonitor := pkg.ApiMonitor{clusterInfo.ApiEndpoint} + + generator := infra.GetBootConfigAssembler(clusterInfo.InfraDriver) + initConfig := generator.Assemble(generateJoinAssets()) + spec := parseSpecFromClusterInfo(clusterInfo) + _ = infraDeployer.Create(spec, initConfig) + apiMonitor.WaitForWorkersReady(0) +} + +func OneShot(filename string){ + clusterInfo := ParseFromFile(filename) + infraDeployer := infra.GetInfraDeployer(clusterInfo.InfraDriver) + apiMonitor := pkg.ApiMonitor{clusterInfo.ApiEndpoint} + + generatePhase := phase.GeneratePhase{"master"} + assets := generatePhase.GenerateAssets() + + generator := infra.GetBootConfigAssembler(clusterInfo.OsType) + masterInitConfig := generator.Assemble(assets) + masterSpec := parseSpecFromClusterInfo(clusterInfo) + _ = infraDeployer.Create(masterSpec, masterInitConfig) + apiMonitor.WaitForMastersReady(0) + + + initConfig := generator.Assemble(generateJoinAssets()) + spec := parseSpecFromClusterInfo(clusterInfo) + _ = infraDeployer.Create(spec, initConfig) + apiMonitor.WaitForWorkersReady(0) + +} + +func parseSpecFromClusterInfo(info ClusterInfo) infra.InfraSpec { + return infra.InfraSpec{} +} + + + +func generateJoinAssets() infra.Assets{ + return infra.Assets{} +} diff --git a/pkg/infra/assets.go b/pkg/infra/assets.go new file mode 100755 index 0000000..8ebcc7e --- /dev/null +++ b/pkg/infra/assets.go @@ -0,0 +1,16 @@ +package infra + +// path : contents +type Assets map[string][]byte + +func (a Assets) ToDir(dirname string) error { + return nil +} + +func (a *Assets) Merge(b Assets) *Assets { + return a +} + +type AssetsGenerator interface { + GenerateAssets() Assets +} \ No newline at end of file diff --git a/pkg/infra/factory.go b/pkg/infra/factory.go new file mode 100755 index 0000000..eb0c340 --- /dev/null +++ b/pkg/infra/factory.go @@ -0,0 +1,18 @@ +package infra + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/bootconfig" + +func GetInfraDeployer(driverType string) InfraDeployer { + if driverType == "openstack"{ + return OpenstackDeployer{} + } + return nil +} + + +func GetBootConfigAssembler(osType string) BootConfigAssembler { + if osType == "NestOS"{ + return bootconfig.IgnitionAssembler{} + } + return nil +} \ No newline at end of file diff --git a/pkg/infra/infradeployer.go b/pkg/infra/infradeployer.go new file mode 100755 index 0000000..d666ce6 --- /dev/null +++ b/pkg/infra/infradeployer.go @@ -0,0 +1,17 @@ +package infra + +type InfraSpec struct{ + diskSize string + memorySize string + image string + clusterCIDR string + initFileContent string +} + + +type InfraDeployer interface { + Create(spec InfraSpec, config InitConfig) error +} + + + diff --git a/pkg/infra/initconfig.go b/pkg/infra/initconfig.go new file mode 100755 index 0000000..97c4bde --- /dev/null +++ b/pkg/infra/initconfig.go @@ -0,0 +1,10 @@ +package infra + +type InitConfig struct{ + OsType string +} + + +type BootConfigAssembler interface { + Assemble(assets Assets) InitConfig +} diff --git a/pkg/infra/openstack.go b/pkg/infra/openstack.go new file mode 100755 index 0000000..393a880 --- /dev/null +++ b/pkg/infra/openstack.go @@ -0,0 +1,10 @@ +package infra + + +type OpenstackDeployer struct { +} + + +func (t OpenstackDeployer) Create(spec InfraSpec, config InitConfig) error{ + return nil +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go new file mode 100644 index 0000000..e69de29 diff --git a/pkg/osimage/osimage.go b/pkg/osimage/osimage.go new file mode 100644 index 0000000..e69de29 diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go new file mode 100644 index 0000000..e69de29 -- Gitee From bd6296c2772cd8071eea0b232a148cb656870721 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Fri, 16 Jun 2023 15:33:09 +0800 Subject: [PATCH 02/38] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E5=BC=80=E6=BA=90=E8=AE=B8=E5=8F=AF=E8=AF=81Apache-2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LICENSE | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file -- Gitee From 734aff969fb6c236f715ca64c37720910182c86b Mon Sep 17 00:00:00 2001 From: lauk Date: Mon, 19 Jun 2023 14:33:14 +0800 Subject: [PATCH 03/38] add operator-manager source code for housekeeper --- housekeeper/Makefile | 79 +++++ housekeeper/PROJECT | 14 + housekeeper/api/v1alpha1/groupversion_info.go | 36 +++ housekeeper/api/v1alpha1/update_types.go | 64 ++++ .../api/v1alpha1/zz_generated.deepcopy.go | 115 ++++++++ .../controllers/update_controller.go | 160 ++++++++++ housekeeper/cmd/housekeeper-operator/main.go | 91 ++++++ .../config/crd/housekeeper.io_updates.yaml | 58 ++++ housekeeper/config/manager/manager.yaml | 42 +++ housekeeper/config/rbac/role.yaml | 63 ++++ housekeeper/config/rbac/role_binding.yaml | 12 + .../housekeeper.io_v1alpha1_update.yaml | 8 + housekeeper/daemon/daemon.go | 0 housekeeper/go.mod | 69 +++++ housekeeper/go.sum | 274 ++++++++++++++++++ housekeeper/operator/operator.go | 0 housekeeper/pkg/common/common.go | 38 +++ housekeeper/pkg/constants/constants.go | 25 ++ housekeeper/pkg/version/version.go | 19 ++ 19 files changed, 1167 insertions(+) create mode 100644 housekeeper/Makefile create mode 100644 housekeeper/PROJECT create mode 100644 housekeeper/api/v1alpha1/groupversion_info.go create mode 100644 housekeeper/api/v1alpha1/update_types.go create mode 100644 housekeeper/api/v1alpha1/zz_generated.deepcopy.go create mode 100644 housekeeper/cmd/housekeeper-operator/controllers/update_controller.go create mode 100644 housekeeper/cmd/housekeeper-operator/main.go create mode 100644 housekeeper/config/crd/housekeeper.io_updates.yaml create mode 100644 housekeeper/config/manager/manager.yaml create mode 100644 housekeeper/config/rbac/role.yaml create mode 100644 housekeeper/config/rbac/role_binding.yaml create mode 100644 housekeeper/config/samples/housekeeper.io_v1alpha1_update.yaml delete mode 100644 housekeeper/daemon/daemon.go create mode 100644 housekeeper/go.mod create mode 100644 housekeeper/go.sum delete mode 100644 housekeeper/operator/operator.go create mode 100644 housekeeper/pkg/common/common.go create mode 100644 housekeeper/pkg/constants/constants.go create mode 100644 housekeeper/pkg/version/version.go diff --git a/housekeeper/Makefile b/housekeeper/Makefile new file mode 100644 index 0000000..5501841 --- /dev/null +++ b/housekeeper/Makefile @@ -0,0 +1,79 @@ +# Copyright 2023 KylinSoft Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +IMG_OPERATOR ?= housekeeper-operator:latest + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +CONTROLLER_TOOLS_VERSION ?= v0.9.2 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + + +##@ Build +.PHONY: all +all: housekeeper-operator-manager + +# Build binary +housekeeper-operator-manager: + go build -o bin/housekeeper-operator-manager cmd/housekeeper-operator/main.go + +# Build the docker image +.PHONY: docker-build +docker-build: ## Build docker image with the housekeeper-operator-manager. + docker build -t ${IMG_OPERATOR} . + +.PHONY: docker-push +docker-push: ## Push docker image with the housekeeper-operator-manager. + docker push ${IMG_OPERATOR} + +# ##@ Development +.PHONY: manifests +manifests: controller-gen ##Generate manifests e.g. CRD/RBAC + $(CONTROLLER_GEN) rbac:roleName=update-manager-role crd paths="./..." output:crd:artifacts:config=config/crd + +.PHONY: install +install: ## Install CRD in a cluster + kubectl apply -f config/crd + +.PHONY: uninstall +uninstall: ## Uninstall CRD from a cluster + kubectl delete -f config/crd + +.PHONY: deploy +deploy: ## Deploy controller + kubectl apply -f config/rbac + kubectl apply -f config/manager + +.PHONY: undeploy +undeploy: ## Undeploy controller + kubectl delete -f config/rbac + kubectl delete -f config/manager + +## Location to install dependencies to +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) diff --git a/housekeeper/PROJECT b/housekeeper/PROJECT new file mode 100644 index 0000000..3abbae3 --- /dev/null +++ b/housekeeper/PROJECT @@ -0,0 +1,14 @@ +layout: +- go.kubebuilder.io/v3 +projectName: housekeeper +repo: housekeeper.io +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + group: housekeeper.io + kind: Update + path: housekeeper.io/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/housekeeper/api/v1alpha1/groupversion_info.go b/housekeeper/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..cd70622 --- /dev/null +++ b/housekeeper/api/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the housekeeper.io v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=housekeeper.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "housekeeper.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/housekeeper/api/v1alpha1/update_types.go b/housekeeper/api/v1alpha1/update_types.go new file mode 100644 index 0000000..7351fd4 --- /dev/null +++ b/housekeeper/api/v1alpha1/update_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// UpdateSpec defines the desired state of Update +type UpdateSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + OSVersion string `json:"osVersion"` + OSImageURL string `json:"osImageURL"` + KubeVersion string `json:"kubeVersion"` +} + +// UpdateStatus defines the observed state of Update +type UpdateStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// Update is the Schema for the updates API +type Update struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec UpdateSpec `json:"spec,omitempty"` + Status UpdateStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// UpdateList contains a list of Update +type UpdateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Update `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Update{}, &UpdateList{}) +} diff --git a/housekeeper/api/v1alpha1/zz_generated.deepcopy.go b/housekeeper/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..c7c9517 --- /dev/null +++ b/housekeeper/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,115 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Update) DeepCopyInto(out *Update) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Update. +func (in *Update) DeepCopy() *Update { + if in == nil { + return nil + } + out := new(Update) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Update) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateList) DeepCopyInto(out *UpdateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Update, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateList. +func (in *UpdateList) DeepCopy() *UpdateList { + if in == nil { + return nil + } + out := new(UpdateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *UpdateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateSpec) DeepCopyInto(out *UpdateSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateSpec. +func (in *UpdateSpec) DeepCopy() *UpdateSpec { + if in == nil { + return nil + } + out := new(UpdateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStatus) DeepCopyInto(out *UpdateStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStatus. +func (in *UpdateStatus) DeepCopy() *UpdateStatus { + if in == nil { + return nil + } + out := new(UpdateStatus) + in.DeepCopyInto(out) + return out +} diff --git a/housekeeper/cmd/housekeeper-operator/controllers/update_controller.go b/housekeeper/cmd/housekeeper-operator/controllers/update_controller.go new file mode 100644 index 0000000..fc98c5b --- /dev/null +++ b/housekeeper/cmd/housekeeper-operator/controllers/update_controller.go @@ -0,0 +1,160 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + "github.com/golang/glog" + housekeeperiov1alpha1 "housekeeper.io/api/v1alpha1" + "housekeeper.io/pkg/common" + "housekeeper.io/pkg/constants" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// UpdateReconciler reconciles a Update object +type UpdateReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=housekeeper.io,resources=updates,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=housekeeper.io,resources=updates/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=housekeeper.io,resources=updates/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Update object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile +func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + + if r.Client == nil { + return common.NoRequeue, nil + } + ctx = context.Background() + err := setLabels(ctx, r, req) + if err != nil { + return common.RequeueNow, fmt.Errorf("unable set nodes label: %v", err) + } + return common.RequeueAfter, nil +} + +func setLabels(ctx context.Context, r common.ReadWriterClient, req ctrl.Request) error { + reqUpgrade, err := labels.NewRequirement(constants.LabelUpgrading, selection.DoesNotExist, nil) + if err != nil { + return fmt.Errorf("unable to create upgrade label requirement: %v", err) + } + reqMaster, err := labels.NewRequirement(constants.LabelMaster, selection.Exists, nil) + if err != nil { + return fmt.Errorf("unable to create master label requirement: %v", err) + } + reqNoMaster, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) + if err != nil { + return fmt.Errorf("unable to create non-master label requirement: %v", err) + } + masterNodes, err := getNodes(ctx, r, *reqUpgrade, *reqMaster) + if err != nil { + return fmt.Errorf("unable to get master node list: %v", err) + } + noMasterNodes, err := getNodes(ctx, r, *reqUpgrade, *reqNoMaster) + if err != nil { + return fmt.Errorf("unable to get non-master node list: %v", err) + } + upgradeCompleted, err := assignUpdated(ctx, r, masterNodes, req.NamespacedName) + if err != nil { + return fmt.Errorf("unabel to add the label of the master nodes: %v", err) + } + //Make sure the master upgrade is complete before start upgrading non-master nodes + if upgradeCompleted { + _, err := assignUpdated(ctx, r, noMasterNodes, req.NamespacedName) + if err != nil { + return fmt.Errorf("unabel to add the label of non-master nodes: %v", err) + } + } + return nil +} + +func getNodes(ctx context.Context, r common.ReadWriterClient, reqs ...labels.Requirement) ([]corev1.Node, error) { + var nodeList corev1.NodeList + opts := client.ListOptions{LabelSelector: labels.NewSelector().Add(reqs...)} + if err := r.List(ctx, &nodeList, &opts); err != nil { + return nil, fmt.Errorf("unable to list nodes with requirements: %v", err) + } + return nodeList.Items, nil +} + +// Add the label to nodes +func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []corev1.Node, name types.NamespacedName) (bool, error) { + var upInstance housekeeperiov1alpha1.Update + upgradeNum := -1 + if err := r.Get(ctx, name, &upInstance); err != nil { + return false, fmt.Errorf("unable to get update Instance %v", err) + } + var ( + kubeVersionSpec = upInstance.Spec.KubeVersion + osVersionSpec = upInstance.Spec.OSVersion + ) + // Add the label after kubernetes version upgrade + labelKubeVersion := fmt.Sprintf("%s%s", constants.LabelKubeVersionPrefix, kubeVersionSpec) + if len(osVersionSpec) == 0 && len(kubeVersionSpec) == 0 { + glog.Info("the os version and kube version cannot be all empty") + return false, nil + } + for _, node := range nodeList { + if len(kubeVersionSpec) > 0 { + if _, ok := node.Labels[labelKubeVersion]; ok { + glog.Infof("successfully upgraded the node: %s", node.Name) + upgradeNum++ + continue + } + } else { + if osVersionSpec == node.Status.NodeInfo.OSImage { + continue + } + } + node.Labels[constants.LabelUpgrading] = "" + if err := r.Update(ctx, &node); err != nil { + glog.Errorf("unable to add %s label:%v", node.Name, err) + } + } + if len(kubeVersionSpec) == 0 { + return true, nil + } + return upgradeNum == len(nodeList), nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&housekeeperiov1alpha1.Update{}). + Complete(r) +} diff --git a/housekeeper/cmd/housekeeper-operator/main.go b/housekeeper/cmd/housekeeper-operator/main.go new file mode 100644 index 0000000..4805269 --- /dev/null +++ b/housekeeper/cmd/housekeeper-operator/main.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + housekeeperiov1alpha1 "housekeeper.io/api/v1alpha1" + "housekeeper.io/cmd/housekeeper-operator/controllers" + "housekeeper.io/pkg/version" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(housekeeperiov1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + opts := zap.Options{} + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + HealthProbeBindAddress: "0", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err = (&controllers.UpdateReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Update") + os.Exit(1) + } + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.WithValues("version: ", version.Version).Info("starting housekeeper-operator manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running housekeeper-operator manager") + os.Exit(1) + } +} diff --git a/housekeeper/config/crd/housekeeper.io_updates.yaml b/housekeeper/config/crd/housekeeper.io_updates.yaml new file mode 100644 index 0000000..7abb679 --- /dev/null +++ b/housekeeper/config/crd/housekeeper.io_updates.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: updates.housekeeper.io +spec: + group: housekeeper.io + names: + kind: Update + listKind: UpdateList + plural: updates + singular: update + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Update is the Schema for the updates API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: UpdateSpec defines the desired state of Update + properties: + kubeVersion: + type: string + osImageURL: + type: string + osVersion: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + type: string + required: + - kubeVersion + - osImageURL + - osVersion + type: object + status: + description: UpdateStatus defines the observed state of Update + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/housekeeper/config/manager/manager.yaml b/housekeeper/config/manager/manager.yaml new file mode 100644 index 0000000..50a1616 --- /dev/null +++ b/housekeeper/config/manager/manager.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: housekeeper-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: housekeeper-operator-manager + namespace: housekeeper-system + labels: + control-plane: housekeeper-operator-manager +spec: + selector: + matchLabels: + control-plane: housekeeper-operator-manager + replicas: 1 + template: + metadata: + labels: + control-plane: housekeeper-operator-manager + spec: + containers: + - command: + - /housekeeper-operator-manager + image: housekeeper-operator:latest + name: housekeeper-operator-manager + securityContext: + allowPrivilegeEscalation: false + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" \ No newline at end of file diff --git a/housekeeper/config/rbac/role.yaml b/housekeeper/config/rbac/role.yaml new file mode 100644 index 0000000..8d34513 --- /dev/null +++ b/housekeeper/config/rbac/role.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: update-manager-role +rules: +- apiGroups: + - housekeeper.io + resources: + - updates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - housekeeper.io + resources: + - updates/finalizers + verbs: + - update +- apiGroups: + - housekeeper.io + resources: + - updates/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - delete + - get diff --git a/housekeeper/config/rbac/role_binding.yaml b/housekeeper/config/rbac/role_binding.yaml new file mode 100644 index 0000000..5ac01f2 --- /dev/null +++ b/housekeeper/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: update-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: update-manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: housekeeper-system diff --git a/housekeeper/config/samples/housekeeper.io_v1alpha1_update.yaml b/housekeeper/config/samples/housekeeper.io_v1alpha1_update.yaml new file mode 100644 index 0000000..b6106bb --- /dev/null +++ b/housekeeper/config/samples/housekeeper.io_v1alpha1_update.yaml @@ -0,0 +1,8 @@ +apiVersion: housekeeper.io/v1alpha1 +kind: Update +metadata: + name: update-sample +spec: + osVersion: os.version + osImageURL: image.url + kubeVersion: kubernetes.version diff --git a/housekeeper/daemon/daemon.go b/housekeeper/daemon/daemon.go deleted file mode 100644 index e69de29..0000000 diff --git a/housekeeper/go.mod b/housekeeper/go.mod new file mode 100644 index 0000000..e256b41d --- /dev/null +++ b/housekeeper/go.mod @@ -0,0 +1,69 @@ +module housekeeper.io + +go 1.19 + +require ( + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + k8s.io/api v0.27.2 + k8s.io/apimachinery v0.27.2 + k8s.io/client-go v0.27.2 + sigs.k8s.io/controller-runtime v0.15.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.27.2 // indirect + k8s.io/component-base v0.27.2 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/housekeeper/go.sum b/housekeeper/go.sum new file mode 100644 index 0000000..4bd65a2 --- /dev/null +++ b/housekeeper/go.sum @@ -0,0 +1,274 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= +gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.27.2 h1:+H17AJpUMvl+clT+BPnKf0E3ksMAzoBBg7CntpSuADo= +k8s.io/api v0.27.2/go.mod h1:ENmbocXfBT2ADujUXcBhHV55RIT31IIEvkntP6vZKS4= +k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo= +k8s.io/apiextensions-apiserver v0.27.2/go.mod h1:Oz9UdvGguL3ULgRdY9QMUzL2RZImotgxvGjdWRq6ZXQ= +k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= +k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/client-go v0.27.2 h1:vDLSeuYvCHKeoQRhCXjxXO45nHVv2Ip4Fe0MfioMrhE= +k8s.io/client-go v0.27.2/go.mod h1:tY0gVmUsHrAmjzHX9zs7eCjxcBsf8IiNe7KQ52biTcQ= +k8s.io/component-base v0.27.2 h1:neju+7s/r5O4x4/txeUONNTS9r1HsPbyoPBAtHsDCpo= +k8s.io/component-base v0.27.2/go.mod h1:5UPk7EjfgrfgRIuDBFtsEFAe4DAvP3U+M8RTzoSJkpo= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= +sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/housekeeper/operator/operator.go b/housekeeper/operator/operator.go deleted file mode 100644 index e69de29..0000000 diff --git a/housekeeper/pkg/common/common.go b/housekeeper/pkg/common/common.go new file mode 100644 index 0000000..ec07e07 --- /dev/null +++ b/housekeeper/pkg/common/common.go @@ -0,0 +1,38 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package common + +import ( + "time" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ReadWriterClient is Kubernetes API +type ReadWriterClient interface { + client.Reader + client.StatusClient + client.Writer +} + +var ( + // controller do not requeue + NoRequeue = ctrl.Result{} + // controller requeue + RequeueNow = ctrl.Result{Requeue: true} + RequeueAfter = ctrl.Result{Requeue: true, RequeueAfter: time.Second * 20} +) diff --git a/housekeeper/pkg/constants/constants.go b/housekeeper/pkg/constants/constants.go new file mode 100644 index 0000000..7ad57f4 --- /dev/null +++ b/housekeeper/pkg/constants/constants.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package constants + +const ( + // LabelUpgrading is the key of the upgrading label for nodes + LabelUpgrading = "upgrade.housekeeper.io/upgrading" + // LabelKubeVersionPrefix defines the label associated with kubernetes version + LabelKubeVersionPrefix = "upgrade.kubernetes.version.io/" + // LabelMaster defines the label associated with master node. + LabelMaster = "node-role.kubernetes.io/master" +) diff --git a/housekeeper/pkg/version/version.go b/housekeeper/pkg/version/version.go new file mode 100644 index 0000000..737ec7b --- /dev/null +++ b/housekeeper/pkg/version/version.go @@ -0,0 +1,19 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package version + +// the version +var Version string = "1.0.0" -- Gitee From c388ec4bed41ae0e0ca90b1ae38f9ab0ec0ac5b6 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Tue, 20 Jun 2023 10:45:11 +0800 Subject: [PATCH 04/38] =?UTF-8?q?=E6=B7=BB=E5=8A=A0infra-provider=E5=8A=9F?= =?UTF-8?q?=E8=83=BD,=E4=B8=BB=E8=A6=81=E5=AE=9E=E7=8E=B0terraform?= =?UTF-8?q?=E7=9A=84init,apply=E5=92=8Cdestroy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 9 +++- go.sum | 35 ++++++++++++++ pkg/cert/etcd.go | 20 ++++++-- pkg/infra/assets.go | 18 ++++++- pkg/infra/factory.go | 23 +++++++-- pkg/infra/infradeployer.go | 35 ++++++++++---- pkg/infra/initconfig.go | 19 +++++++- pkg/infra/openstack.go | 45 ++++++++++++++++- pkg/infra/terraform/logger.go | 27 +++++++++++ pkg/infra/terraform/state.go | 46 ++++++++++++++++++ pkg/infra/terraform/terraform.go | 83 ++++++++++++++++++++++++++++++++ 11 files changed, 338 insertions(+), 22 deletions(-) mode change 100755 => 100644 go.mod mode change 100755 => 100644 go.sum create mode 100644 pkg/infra/terraform/logger.go create mode 100644 pkg/infra/terraform/state.go create mode 100644 pkg/infra/terraform/terraform.go diff --git a/go.mod b/go.mod old mode 100755 new mode 100644 index 0fe9ed1..ca5e20f --- a/go.mod +++ b/go.mod @@ -1,14 +1,21 @@ module gitee.com/openeuler/nestos-kubernetes-deployer -go 1.18 +go 1.19 require ( + github.com/hashicorp/terraform-exec v0.18.1 github.com/lithammer/dedent v1.1.0 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.7.0 ) require ( + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/terraform-json v0.16.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/zclconf/go-cty v1.13.1 // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/text v0.8.0 // indirect ) diff --git a/go.sum b/go.sum old mode 100755 new mode 100644 index 1516f81..ec697df --- a/go.sum +++ b/go.sum @@ -1,14 +1,49 @@ +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hc-install v0.5.0 h1:D9bl4KayIYKEeJ4vUDe9L5huqxZXczKaykSRcmQ0xY0= +github.com/hashicorp/terraform-exec v0.18.1 h1:LAbfDvNQU1l0NOQlTuudjczVhHj061fNX5H8XZxHlH4= +github.com/hashicorp/terraform-exec v0.18.1/go.mod h1:58wg4IeuAJ6LVsLUeD2DWZZoc/bYi6dzhLHzxM41980= +github.com/hashicorp/terraform-json v0.16.0 h1:UKkeWRWb23do5LNAFlh/K3N0ymn1qTOO8c+85Albo3s= +github.com/hashicorp/terraform-json v0.16.0/go.mod h1:v0Ufk9jJnk6tcIZvScHvetlKfiNTC+WS21mnXIlc0B0= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/zclconf/go-cty v1.13.1 h1:0a6bRwuiSHtAmqCqNOE+c2oHgepv0ctoxU4FUe43kwc= +github.com/zclconf/go-cty v1.13.1/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/cert/etcd.go b/pkg/cert/etcd.go index 93cc24f..57a64f5 100755 --- a/pkg/cert/etcd.go +++ b/pkg/cert/etcd.go @@ -1,11 +1,25 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package cert import ( "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra" ) - - type EtcdCaGenerator struct { } @@ -14,4 +28,4 @@ func (e EtcdCaGenerator) GenerateAssets() infra.Assets { } func init() { -} \ No newline at end of file +} diff --git a/pkg/infra/assets.go b/pkg/infra/assets.go index 8ebcc7e..176dd31 100755 --- a/pkg/infra/assets.go +++ b/pkg/infra/assets.go @@ -1,3 +1,19 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package infra // path : contents @@ -13,4 +29,4 @@ func (a *Assets) Merge(b Assets) *Assets { type AssetsGenerator interface { GenerateAssets() Assets -} \ No newline at end of file +} diff --git a/pkg/infra/factory.go b/pkg/infra/factory.go index eb0c340..3db63de 100755 --- a/pkg/infra/factory.go +++ b/pkg/infra/factory.go @@ -1,18 +1,33 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package infra import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/bootconfig" func GetInfraDeployer(driverType string) InfraDeployer { - if driverType == "openstack"{ + if driverType == "openstack" { return OpenstackDeployer{} } return nil } - func GetBootConfigAssembler(osType string) BootConfigAssembler { - if osType == "NestOS"{ + if osType == "NestOS" { return bootconfig.IgnitionAssembler{} } return nil -} \ No newline at end of file +} diff --git a/pkg/infra/infradeployer.go b/pkg/infra/infradeployer.go index d666ce6..f2c42b4 100755 --- a/pkg/infra/infradeployer.go +++ b/pkg/infra/infradeployer.go @@ -1,17 +1,32 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package infra -type InfraSpec struct{ - diskSize string - memorySize string - image string - clusterCIDR string +import "github.com/hashicorp/terraform-exec/tfexec" + +type InfraSpec struct { + diskSize string + memorySize string + image string + clusterCIDR string initFileContent string } - type InfraDeployer interface { - Create(spec InfraSpec, config InitConfig) error + Create(spec InfraSpec, config InitConfig, extraOpts ...tfexec.ApplyOption) error + Destroy(spec InfraSpec, config InitConfig, extraOpts ...tfexec.DestroyOption) error } - - - diff --git a/pkg/infra/initconfig.go b/pkg/infra/initconfig.go index 97c4bde..288c3b3 100755 --- a/pkg/infra/initconfig.go +++ b/pkg/infra/initconfig.go @@ -1,10 +1,25 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package infra -type InitConfig struct{ +type InitConfig struct { OsType string } - type BootConfigAssembler interface { Assemble(assets Assets) InitConfig } diff --git a/pkg/infra/openstack.go b/pkg/infra/openstack.go index 393a880..be3bc69 100755 --- a/pkg/infra/openstack.go +++ b/pkg/infra/openstack.go @@ -1,10 +1,53 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package infra +import ( + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" + "github.com/hashicorp/terraform-exec/tfexec" +) type OpenstackDeployer struct { + tfFilePath string + terraformBinary string +} + +func (t OpenstackDeployer) Create(extraOpts ...tfexec.ApplyOption) error { + if err := terraform.Init(t.tfFilePath, t.terraformBinary); err != nil { + return err + } + + applyErr := terraform.Apply(t.tfFilePath, t.terraformBinary, extraOpts...) + if applyErr != nil { + return applyErr + } + + return nil } +func (t OpenstackDeployer) Destroy(extraOpts ...tfexec.DestroyOption) error { + if err := terraform.Init(t.tfFilePath, t.terraformBinary); err != nil { + return err + } + + destroyErr := terraform.Destroy(t.tfFilePath, t.terraformBinary, extraOpts) + if destroyErr != nil { + return destroyErr + } -func (t OpenstackDeployer) Create(spec InfraSpec, config InitConfig) error{ return nil } diff --git a/pkg/infra/terraform/logger.go b/pkg/infra/terraform/logger.go new file mode 100644 index 0000000..bfb609d --- /dev/null +++ b/pkg/infra/terraform/logger.go @@ -0,0 +1,27 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package terraform + +type printfer struct { +} + +func newPrintfer() *printfer { + return &printfer{} +} + +func (p *printfer) Printf(format string, ifs ...interface{}) { +} diff --git a/pkg/infra/terraform/state.go b/pkg/infra/terraform/state.go new file mode 100644 index 0000000..fd7481e --- /dev/null +++ b/pkg/infra/terraform/state.go @@ -0,0 +1,46 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package terraform + +import ( + "context" + + "github.com/pkg/errors" +) + +func Outputs(tfFilePath string, terraformBinary string) error { + // TODO 解析tfstate文件，读取特定内容 + + tf, err := newTFExec(tfFilePath, terraformBinary) + if err != nil { + return err + } + + tfoutput, err := tf.Output(context.Background()) + if err != nil { + return errors.Wrap(err, "failed to read terraform state file") + } + + outputs := make(map[string]interface{}, len(tfoutput)) + for key, value := range tfoutput { + outputs[key] = value.Value + } + + // TODO 解析outputs + + return nil +} diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go new file mode 100644 index 0000000..7c387a7 --- /dev/null +++ b/pkg/infra/terraform/terraform.go @@ -0,0 +1,83 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package terraform + +import ( + "context" + "path/filepath" + + "github.com/hashicorp/terraform-exec/tfexec" + "github.com/pkg/errors" +) + +func newTFExec(tfFilePath string, terraformBinary string) (*tfexec.Terraform, error) { + tf, err := tfexec.NewTerraform(tfFilePath, terraformBinary) + if err != nil { + return nil, err + } + + // TODO 日志输出 + // tf.SetStdout() + // tf.SetStderr() + // tf.SetLogger(newPrintfer()) + + return tf, nil +} + +func Init(tfFilePath string, terraformBinary string) error { + tf, err := newTFExec(tfFilePath, terraformBinary) + if err != nil { + return errors.Wrap(err, "failed to create a new tfexec.") + } + + // 如果想导入本地已有插件，需手动构建相应目录 + // 如openstack插件所需目录为plugins/registry.terraform.io/terraform-provider-openstack/openstack/1.51.1/linux_arm64/terraform-provider-openstack_v1.51.1 + return errors.Wrap( + tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformBinary, "plugins"))), + "failed doing terraform init.", + ) +} + +func Apply(tfFilePath string, terraformBinary string, extraOpts ...tfexec.ApplyOption) error { + tf, err := newTFExec(tfFilePath, terraformBinary) + if err != nil { + return errors.Wrap(err, "failed to create a new tfexec.") + } + + // TODO 日志输出 + // tf.SetStdout() + // tf.SetStderr() + tf.SetLogger(newPrintfer()) + + err = tf.Apply(context.Background(), extraOpts...) + return errors.Wrap(err, "failed to apply Terraform.") +} + +func Destroy(tfFilePath string, terraformBinary string, extraOpts ...tfexec.DestroyOption) error { + tf, err := newTFExec(tfFilePath, terraformBinary) + if err != nil { + return errors.Wrap(err, "failed to create a new tfexec.") + } + + // TODO 日志输出 + // tf.SetStdout() + // tf.SetStderr() + tf.SetLogger(newPrintfer()) + + err = tf.Destroy(context.Background(), extraOpts...) + return errors.Wrap(err, "failed to apply Terraform.") +} -- Gitee From 85100c38b1782cb0174a08b06052b0f3c04704b6 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Wed, 21 Jun 2023 09:59:44 +0800 Subject: [PATCH 05/38] =?UTF-8?q?=E5=AE=8C=E5=96=84terraform=20init?= =?UTF-8?q?=E5=87=BD=E6=95=B0tfInit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/infra/terraform/terraform.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index 7c387a7..1381d82 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -38,8 +38,14 @@ func newTFExec(tfFilePath string, terraformBinary string) (*tfexec.Terraform, er return tf, nil } -func Init(tfFilePath string, terraformBinary string) error { - tf, err := newTFExec(tfFilePath, terraformBinary) +// terraform init +func tfInit(dir string, platform string, target string, terraformDir string, providers []prov.Provider) (err error) { + err = unpack(dir, platform, target) + if err != nil { + return errors.Wrap(err, "failed to unpack Terraform modules") + } + + tf, err := newTFExec(dir, terraformDir) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") } @@ -47,12 +53,12 @@ func Init(tfFilePath string, terraformBinary string) error { // 如果想导入本地已有插件，需手动构建相应目录 // 如openstack插件所需目录为plugins/registry.terraform.io/terraform-provider-openstack/openstack/1.51.1/linux_arm64/terraform-provider-openstack_v1.51.1 return errors.Wrap( - tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformBinary, "plugins"))), + tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformDir, "plugins"))), "failed doing terraform init.", ) } -func Apply(tfFilePath string, terraformBinary string, extraOpts ...tfexec.ApplyOption) error { +func tfApply(tfFilePath string, terraformBinary string, extraOpts ...tfexec.ApplyOption) error { tf, err := newTFExec(tfFilePath, terraformBinary) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") @@ -67,7 +73,7 @@ func Apply(tfFilePath string, terraformBinary string, extraOpts ...tfexec.ApplyO return errors.Wrap(err, "failed to apply Terraform.") } -func Destroy(tfFilePath string, terraformBinary string, extraOpts ...tfexec.DestroyOption) error { +func tfDestroy(tfFilePath string, terraformBinary string, extraOpts ...tfexec.DestroyOption) error { tf, err := newTFExec(tfFilePath, terraformBinary) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") -- Gitee From 3beb3299ea79c4fb0460382e5a83fd9c456a3704 Mon Sep 17 00:00:00 2001 From: lauk Date: Tue, 20 Jun 2023 15:49:34 +0800 Subject: [PATCH 06/38] add controller-manager source code for housekeeper --- housekeeper/Makefile | 10 +- housekeeper/go.mod | 6 +- housekeeper/go.sum | 47 ++- housekeeper/{ => operator}/PROJECT | 0 .../api/v1alpha1/groupversion_info.go | 0 .../api/v1alpha1/update_types.go | 0 .../api/v1alpha1/zz_generated.deepcopy.go | 0 .../config/crd/housekeeper.io_updates.yaml | 5 +- .../config/manager/manager.yaml | 2 +- .../{ => operator}/config/rbac/role.yaml | 0 .../config/rbac/role_binding.yaml | 0 .../housekeeper.io_v1alpha1_update.yaml | 0 .../controllers/update_controller.go | 44 ++- .../housekeeper-operator/main.go | 24 +- housekeeper/pkg/connection/connection.go | 67 ++++ housekeeper/pkg/connection/proto/Makefile | 17 + housekeeper/pkg/connection/proto/daemon.pb.go | 343 ++++++++++++++++++ housekeeper/pkg/connection/proto/daemon.proto | 36 ++ housekeeper/pkg/constants/constants.go | 7 + 19 files changed, 569 insertions(+), 39 deletions(-) rename housekeeper/{ => operator}/PROJECT (100%) rename housekeeper/{ => operator}/api/v1alpha1/groupversion_info.go (100%) rename housekeeper/{ => operator}/api/v1alpha1/update_types.go (100%) rename housekeeper/{ => operator}/api/v1alpha1/zz_generated.deepcopy.go (100%) rename housekeeper/{ => operator}/config/crd/housekeeper.io_updates.yaml (91%) rename housekeeper/{ => operator}/config/manager/manager.yaml (97%) rename housekeeper/{ => operator}/config/rbac/role.yaml (100%) rename housekeeper/{ => operator}/config/rbac/role_binding.yaml (100%) rename housekeeper/{ => operator}/config/samples/housekeeper.io_v1alpha1_update.yaml (100%) rename housekeeper/{cmd => operator}/housekeeper-operator/controllers/update_controller.go (79%) rename housekeeper/{cmd => operator}/housekeeper-operator/main.go (77%) create mode 100644 housekeeper/pkg/connection/connection.go create mode 100644 housekeeper/pkg/connection/proto/Makefile create mode 100644 housekeeper/pkg/connection/proto/daemon.pb.go create mode 100644 housekeeper/pkg/connection/proto/daemon.proto diff --git a/housekeeper/Makefile b/housekeeper/Makefile index 5501841..590e7f6 100644 --- a/housekeeper/Makefile +++ b/housekeeper/Makefile @@ -13,6 +13,7 @@ # limitations under the License. IMG_OPERATOR ?= housekeeper-operator:latest +IMG_CONTROLLER ?= housekeeper-controller:latest ## Location to install dependencies to LOCALBIN ?= $(shell pwd)/bin @@ -34,20 +35,23 @@ SHELL = /usr/bin/env bash -o pipefail ##@ Build .PHONY: all -all: housekeeper-operator-manager - +all: housekeeper-operator-manager housekeeper-controller-manager # Build binary housekeeper-operator-manager: - go build -o bin/housekeeper-operator-manager cmd/housekeeper-operator/main.go + go build -o bin/housekeeper-operator-manager operator/housekeeper-operator/main.go +housekeeper-controller-manager: + go build -o bin/housekeeper-controller-manager operator/housekeeper-controller/main.go # Build the docker image .PHONY: docker-build docker-build: ## Build docker image with the housekeeper-operator-manager. docker build -t ${IMG_OPERATOR} . + docker build -t ${IMG_CONTROLLER} . .PHONY: docker-push docker-push: ## Push docker image with the housekeeper-operator-manager. docker push ${IMG_OPERATOR} + docker push ${IMG_CONTROLLER} # ##@ Development .PHONY: manifests diff --git a/housekeeper/go.mod b/housekeeper/go.mod index e256b41d..ed1b7c4 100644 --- a/housekeeper/go.mod +++ b/housekeeper/go.mod @@ -3,7 +3,9 @@ module housekeeper.io go 1.19 require ( - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/sirupsen/logrus v1.9.0 + google.golang.org/grpc v1.51.0 + google.golang.org/protobuf v1.30.0 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.2 k8s.io/client-go v0.27.2 @@ -54,7 +56,7 @@ require ( golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/housekeeper/go.sum b/housekeeper/go.sum index 4bd65a2..ed4a5d6 100644 --- a/housekeeper/go.sum +++ b/housekeeper/go.sum @@ -1,13 +1,22 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -15,13 +24,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -36,7 +50,6 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -44,12 +57,15 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -60,15 +76,19 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -113,7 +133,10 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -130,6 +153,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -151,17 +175,20 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -174,10 +201,13 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -187,6 +217,7 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -213,21 +244,34 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -238,6 +282,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/housekeeper/PROJECT b/housekeeper/operator/PROJECT similarity index 100% rename from housekeeper/PROJECT rename to housekeeper/operator/PROJECT diff --git a/housekeeper/api/v1alpha1/groupversion_info.go b/housekeeper/operator/api/v1alpha1/groupversion_info.go similarity index 100% rename from housekeeper/api/v1alpha1/groupversion_info.go rename to housekeeper/operator/api/v1alpha1/groupversion_info.go diff --git a/housekeeper/api/v1alpha1/update_types.go b/housekeeper/operator/api/v1alpha1/update_types.go similarity index 100% rename from housekeeper/api/v1alpha1/update_types.go rename to housekeeper/operator/api/v1alpha1/update_types.go diff --git a/housekeeper/api/v1alpha1/zz_generated.deepcopy.go b/housekeeper/operator/api/v1alpha1/zz_generated.deepcopy.go similarity index 100% rename from housekeeper/api/v1alpha1/zz_generated.deepcopy.go rename to housekeeper/operator/api/v1alpha1/zz_generated.deepcopy.go diff --git a/housekeeper/config/crd/housekeeper.io_updates.yaml b/housekeeper/operator/config/crd/housekeeper.io_updates.yaml similarity index 91% rename from housekeeper/config/crd/housekeeper.io_updates.yaml rename to housekeeper/operator/config/crd/housekeeper.io_updates.yaml index 7abb679..53cf004 100644 --- a/housekeeper/config/crd/housekeeper.io_updates.yaml +++ b/housekeeper/operator/config/crd/housekeeper.io_updates.yaml @@ -36,12 +36,13 @@ spec: description: UpdateSpec defines the desired state of Update properties: kubeVersion: + description: 'The version used to upgrade k8s' type: string osImageURL: + description: 'The image url used to upgrade OS' type: string osVersion: - description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - Important: Run "make" to regenerate code after modifying this file' + description: 'The version used to upgrade OS' type: string required: - kubeVersion diff --git a/housekeeper/config/manager/manager.yaml b/housekeeper/operator/config/manager/manager.yaml similarity index 97% rename from housekeeper/config/manager/manager.yaml rename to housekeeper/operator/config/manager/manager.yaml index 50a1616..7544d7c 100644 --- a/housekeeper/config/manager/manager.yaml +++ b/housekeeper/operator/config/manager/manager.yaml @@ -39,4 +39,4 @@ spec: node-role.kubernetes.io/control-plane: "" tolerations: - key: "node-role.kubernetes.io/master" - operator: "Exists" \ No newline at end of file + operator: "Exists" diff --git a/housekeeper/config/rbac/role.yaml b/housekeeper/operator/config/rbac/role.yaml similarity index 100% rename from housekeeper/config/rbac/role.yaml rename to housekeeper/operator/config/rbac/role.yaml diff --git a/housekeeper/config/rbac/role_binding.yaml b/housekeeper/operator/config/rbac/role_binding.yaml similarity index 100% rename from housekeeper/config/rbac/role_binding.yaml rename to housekeeper/operator/config/rbac/role_binding.yaml diff --git a/housekeeper/config/samples/housekeeper.io_v1alpha1_update.yaml b/housekeeper/operator/config/samples/housekeeper.io_v1alpha1_update.yaml similarity index 100% rename from housekeeper/config/samples/housekeeper.io_v1alpha1_update.yaml rename to housekeeper/operator/config/samples/housekeeper.io_v1alpha1_update.yaml diff --git a/housekeeper/cmd/housekeeper-operator/controllers/update_controller.go b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go similarity index 79% rename from housekeeper/cmd/housekeeper-operator/controllers/update_controller.go rename to housekeeper/operator/housekeeper-operator/controllers/update_controller.go index fc98c5b..1346a26 100644 --- a/housekeeper/cmd/housekeeper-operator/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - "github.com/golang/glog" - housekeeperiov1alpha1 "housekeeper.io/api/v1alpha1" + "github.com/sirupsen/logrus" + housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" "housekeeper.io/pkg/common" "housekeeper.io/pkg/constants" @@ -63,7 +63,8 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr ctx = context.Background() err := setLabels(ctx, r, req) if err != nil { - return common.RequeueNow, fmt.Errorf("unable set nodes label: %v", err) + logrus.Errorf("unable set nodes label: %v", err) + return common.RequeueNow, err } return common.RequeueAfter, nil } @@ -71,33 +72,40 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr func setLabels(ctx context.Context, r common.ReadWriterClient, req ctrl.Request) error { reqUpgrade, err := labels.NewRequirement(constants.LabelUpgrading, selection.DoesNotExist, nil) if err != nil { - return fmt.Errorf("unable to create upgrade label requirement: %v", err) + logrus.Errorf("unable to create upgrade label requirement: %v", err) + return err } reqMaster, err := labels.NewRequirement(constants.LabelMaster, selection.Exists, nil) if err != nil { - return fmt.Errorf("unable to create master label requirement: %v", err) + logrus.Errorf("unable to create master label requirement: %v", err) + return err } reqNoMaster, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) if err != nil { - return fmt.Errorf("unable to create non-master label requirement: %v", err) + logrus.Errorf("unable to create non-master label requirement: %v", err) + return err } masterNodes, err := getNodes(ctx, r, *reqUpgrade, *reqMaster) if err != nil { - return fmt.Errorf("unable to get master node list: %v", err) + logrus.Errorf("unable to get master node list: %v", err) + return err } noMasterNodes, err := getNodes(ctx, r, *reqUpgrade, *reqNoMaster) if err != nil { - return fmt.Errorf("unable to get non-master node list: %v", err) + logrus.Errorf("unable to get non-master node list: %v", err) + return err } upgradeCompleted, err := assignUpdated(ctx, r, masterNodes, req.NamespacedName) if err != nil { - return fmt.Errorf("unabel to add the label of the master nodes: %v", err) + logrus.Errorf("unabel to add the label of the master nodes: %v", err) + return err } //Make sure the master upgrade is complete before start upgrading non-master nodes if upgradeCompleted { _, err := assignUpdated(ctx, r, noMasterNodes, req.NamespacedName) if err != nil { - return fmt.Errorf("unabel to add the label of non-master nodes: %v", err) + logrus.Errorf("unabel to add the label of non-master nodes: %v", err) + return err } } return nil @@ -107,7 +115,8 @@ func getNodes(ctx context.Context, r common.ReadWriterClient, reqs ...labels.Req var nodeList corev1.NodeList opts := client.ListOptions{LabelSelector: labels.NewSelector().Add(reqs...)} if err := r.List(ctx, &nodeList, &opts); err != nil { - return nil, fmt.Errorf("unable to list nodes with requirements: %v", err) + logrus.Errorf("unable to list nodes with requirements: %v", err) + return nil, err } return nodeList.Items, nil } @@ -117,22 +126,23 @@ func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []co var upInstance housekeeperiov1alpha1.Update upgradeNum := -1 if err := r.Get(ctx, name, &upInstance); err != nil { - return false, fmt.Errorf("unable to get update Instance %v", err) + logrus.Errorf("unable to get update Instance %v", err) + return false, err } var ( kubeVersionSpec = upInstance.Spec.KubeVersion osVersionSpec = upInstance.Spec.OSVersion ) - // Add the label after kubernetes version upgrade + //labelKubeVersion added after kube version upgrade labelKubeVersion := fmt.Sprintf("%s%s", constants.LabelKubeVersionPrefix, kubeVersionSpec) - if len(osVersionSpec) == 0 && len(kubeVersionSpec) == 0 { - glog.Info("the os version and kube version cannot be all empty") + if len(osVersionSpec) == 0 { + logrus.Warning("os version is required") return false, nil } for _, node := range nodeList { if len(kubeVersionSpec) > 0 { if _, ok := node.Labels[labelKubeVersion]; ok { - glog.Infof("successfully upgraded the node: %s", node.Name) + logrus.Infof("successfully upgraded the node: %s", node.Name) upgradeNum++ continue } @@ -143,7 +153,7 @@ func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []co } node.Labels[constants.LabelUpgrading] = "" if err := r.Update(ctx, &node); err != nil { - glog.Errorf("unable to add %s label:%v", node.Name, err) + logrus.Errorf("unable to add %s label:%v", node.Name, err) } } if len(kubeVersionSpec) == 0 { diff --git a/housekeeper/cmd/housekeeper-operator/main.go b/housekeeper/operator/housekeeper-operator/main.go similarity index 77% rename from housekeeper/cmd/housekeeper-operator/main.go rename to housekeeper/operator/housekeeper-operator/main.go index 4805269..f1f66e1 100644 --- a/housekeeper/cmd/housekeeper-operator/main.go +++ b/housekeeper/operator/housekeeper-operator/main.go @@ -22,6 +22,8 @@ import ( // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. + + "github.com/sirupsen/logrus" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/apimachinery/pkg/runtime" @@ -31,16 +33,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - housekeeperiov1alpha1 "housekeeper.io/api/v1alpha1" - "housekeeper.io/cmd/housekeeper-operator/controllers" + housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" + "housekeeper.io/operator/housekeeper-operator/controllers" "housekeeper.io/pkg/version" //+kubebuilder:scaffold:imports ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) +var scheme = runtime.NewScheme() func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -61,7 +60,7 @@ func main() { HealthProbeBindAddress: "0", }) if err != nil { - setupLog.Error(err, "unable to start manager") + logrus.Error(err, "unable to start manager") os.Exit(1) } @@ -69,23 +68,22 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Update") + logrus.Error(err, "unable to create controller", "controller", "Update") os.Exit(1) } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") + logrus.Errorf("unable to set up health check: %v", err) os.Exit(1) } if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + logrus.Errorf("unable to set up ready check: %v", err) os.Exit(1) } - - setupLog.WithValues("version: ", version.Version).Info("starting housekeeper-operator manager") + logrus.Infof("starting housekeeper-operator manager version:", version.Version) if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running housekeeper-operator manager") + logrus.Errorf("problem running housekeeper-operator manager: %v", err) os.Exit(1) } } diff --git a/housekeeper/pkg/connection/connection.go b/housekeeper/pkg/connection/connection.go new file mode 100644 index 0000000..3e5c1c6 --- /dev/null +++ b/housekeeper/pkg/connection/connection.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// connection between client and server +package connection + +import ( + "context" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + + pb "housekeeper.io/pkg/connection/proto" +) + +type Client struct { + socketAddress string + client pb.UpgradeClusterClient +} + +type PushInfo struct { + OSImageURL string + OSVersion string + KubeVersion string + ControlPlane bool +} + +// Create a grpc channel +func New(socketAddr string) (*Client, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + bc := backoff.DefaultConfig + bc.MaxDelay = 5 * time.Second + + connection, err := grpc.DialContext(ctx, socketAddr, grpc.WithInsecure(), grpc.WithBlock(), + grpc.WithConnectParams(grpc.ConnectParams{Backoff: bc})) + if err != nil { + return nil, err + } + return &Client{socketAddress: socketAddr, client: pb.NewUpgradeClusterClient(connection)}, nil +} + +// send update requests +func (c *Client) UpgradeKubeSpec(pushInfo *PushInfo) error { + _, err := c.client.Upgrade(context.Background(), + &pb.UpgradeRequest{ + KubeVersion: pushInfo.KubeVersion, + OsImageUrl: pushInfo.OSImageURL, + OsVersion: pushInfo.OSVersion, + ControlPlane: pushInfo.ControlPlane, + }) + return err +} diff --git a/housekeeper/pkg/connection/proto/Makefile b/housekeeper/pkg/connection/proto/Makefile new file mode 100644 index 0000000..7e8277a --- /dev/null +++ b/housekeeper/pkg/connection/proto/Makefile @@ -0,0 +1,17 @@ +# Copyright 2023 KylinSoft Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +grpc: + protoc --go_out=plugins=grpc:. --go_opt=paths=source_relative daemon.proto diff --git a/housekeeper/pkg/connection/proto/daemon.pb.go b/housekeeper/pkg/connection/proto/daemon.pb.go new file mode 100644 index 0000000..9ef7338 --- /dev/null +++ b/housekeeper/pkg/connection/proto/daemon.pb.go @@ -0,0 +1,343 @@ +// +//Copyright 2023 KylinSoft Co., Ltd. +// +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.20.3 +// source: daemon.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type UpgradeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + KubeVersion string `protobuf:"bytes,1,opt,name=kube_version,json=kubeVersion,proto3" json:"kube_version,omitempty"` + OsImageUrl string `protobuf:"bytes,2,opt,name=os_image_url,json=osImageUrl,proto3" json:"os_image_url,omitempty"` + OsVersion string `protobuf:"bytes,3,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"` + ControlPlane bool `protobuf:"varint,4,opt,name=control_plane,json=controlPlane,proto3" json:"control_plane,omitempty"` +} + +func (x *UpgradeRequest) Reset() { + *x = UpgradeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_daemon_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpgradeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpgradeRequest) ProtoMessage() {} + +func (x *UpgradeRequest) ProtoReflect() protoreflect.Message { + mi := &file_daemon_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpgradeRequest.ProtoReflect.Descriptor instead. +func (*UpgradeRequest) Descriptor() ([]byte, []int) { + return file_daemon_proto_rawDescGZIP(), []int{0} +} + +func (x *UpgradeRequest) GetKubeVersion() string { + if x != nil { + return x.KubeVersion + } + return "" +} + +func (x *UpgradeRequest) GetOsImageUrl() string { + if x != nil { + return x.OsImageUrl + } + return "" +} + +func (x *UpgradeRequest) GetOsVersion() string { + if x != nil { + return x.OsVersion + } + return "" +} + +func (x *UpgradeRequest) GetControlPlane() bool { + if x != nil { + return x.ControlPlane + } + return false +} + +type UpgradeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Err int32 `protobuf:"varint,1,opt,name=err,proto3" json:"err,omitempty"` +} + +func (x *UpgradeResponse) Reset() { + *x = UpgradeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_daemon_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpgradeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpgradeResponse) ProtoMessage() {} + +func (x *UpgradeResponse) ProtoReflect() protoreflect.Message { + mi := &file_daemon_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpgradeResponse.ProtoReflect.Descriptor instead. +func (*UpgradeResponse) Descriptor() ([]byte, []int) { + return file_daemon_proto_rawDescGZIP(), []int{1} +} + +func (x *UpgradeResponse) GetErr() int32 { + if x != nil { + return x.Err + } + return 0 +} + +var File_daemon_proto protoreflect.FileDescriptor + +var file_daemon_proto_rawDesc = []byte{ + 0x0a, 0x0c, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x22, 0x99, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6b, 0x75, 0x62, + 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x6b, 0x75, 0x62, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, + 0x6f, 0x73, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x73, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1d, + 0x0a, 0x0a, 0x6f, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x6f, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, + 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, + 0x6e, 0x65, 0x22, 0x23, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x32, 0x4e, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x07, 0x55, 0x70, 0x67, + 0x72, 0x61, 0x64, 0x65, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x25, 0x5a, 0x23, 0x68, 0x6f, 0x75, 0x73, 0x65, + 0x6b, 0x65, 0x65, 0x70, 0x65, 0x72, 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_daemon_proto_rawDescOnce sync.Once + file_daemon_proto_rawDescData = file_daemon_proto_rawDesc +) + +func file_daemon_proto_rawDescGZIP() []byte { + file_daemon_proto_rawDescOnce.Do(func() { + file_daemon_proto_rawDescData = protoimpl.X.CompressGZIP(file_daemon_proto_rawDescData) + }) + return file_daemon_proto_rawDescData +} + +var file_daemon_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_daemon_proto_goTypes = []interface{}{ + (*UpgradeRequest)(nil), // 0: daemon.UpgradeRequest + (*UpgradeResponse)(nil), // 1: daemon.UpgradeResponse +} +var file_daemon_proto_depIdxs = []int32{ + 0, // 0: daemon.UpgradeCluster.Upgrade:input_type -> daemon.UpgradeRequest + 1, // 1: daemon.UpgradeCluster.Upgrade:output_type -> daemon.UpgradeResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_daemon_proto_init() } +func file_daemon_proto_init() { + if File_daemon_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_daemon_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpgradeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_daemon_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpgradeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_daemon_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_daemon_proto_goTypes, + DependencyIndexes: file_daemon_proto_depIdxs, + MessageInfos: file_daemon_proto_msgTypes, + }.Build() + File_daemon_proto = out.File + file_daemon_proto_rawDesc = nil + file_daemon_proto_goTypes = nil + file_daemon_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// UpgradeClusterClient is the client API for UpgradeCluster service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type UpgradeClusterClient interface { + Upgrade(ctx context.Context, in *UpgradeRequest, opts ...grpc.CallOption) (*UpgradeResponse, error) +} + +type upgradeClusterClient struct { + cc grpc.ClientConnInterface +} + +func NewUpgradeClusterClient(cc grpc.ClientConnInterface) UpgradeClusterClient { + return &upgradeClusterClient{cc} +} + +func (c *upgradeClusterClient) Upgrade(ctx context.Context, in *UpgradeRequest, opts ...grpc.CallOption) (*UpgradeResponse, error) { + out := new(UpgradeResponse) + err := c.cc.Invoke(ctx, "/daemon.UpgradeCluster/Upgrade", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// UpgradeClusterServer is the server API for UpgradeCluster service. +type UpgradeClusterServer interface { + Upgrade(context.Context, *UpgradeRequest) (*UpgradeResponse, error) +} + +// UnimplementedUpgradeClusterServer can be embedded to have forward compatible implementations. +type UnimplementedUpgradeClusterServer struct { +} + +func (*UnimplementedUpgradeClusterServer) Upgrade(context.Context, *UpgradeRequest) (*UpgradeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Upgrade not implemented") +} + +func RegisterUpgradeClusterServer(s *grpc.Server, srv UpgradeClusterServer) { + s.RegisterService(&_UpgradeCluster_serviceDesc, srv) +} + +func _UpgradeCluster_Upgrade_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpgradeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(UpgradeClusterServer).Upgrade(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/daemon.UpgradeCluster/Upgrade", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(UpgradeClusterServer).Upgrade(ctx, req.(*UpgradeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _UpgradeCluster_serviceDesc = grpc.ServiceDesc{ + ServiceName: "daemon.UpgradeCluster", + HandlerType: (*UpgradeClusterServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Upgrade", + Handler: _UpgradeCluster_Upgrade_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "daemon.proto", +} diff --git a/housekeeper/pkg/connection/proto/daemon.proto b/housekeeper/pkg/connection/proto/daemon.proto new file mode 100644 index 0000000..649f714 --- /dev/null +++ b/housekeeper/pkg/connection/proto/daemon.proto @@ -0,0 +1,36 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +syntax = "proto3"; + +option go_package = "housekeeper.io/pkg/connection/proto"; + +package daemon; + +service UpgradeCluster{ + rpc Upgrade(UpgradeRequest) returns (UpgradeResponse) {} +} + +message UpgradeRequest { + string kube_version = 1; + string os_image_url = 2; + string os_version = 3; + bool control_plane = 4; +} + +message UpgradeResponse { + int32 err = 1; +} \ No newline at end of file diff --git a/housekeeper/pkg/constants/constants.go b/housekeeper/pkg/constants/constants.go index 7ad57f4..ea930a4 100644 --- a/housekeeper/pkg/constants/constants.go +++ b/housekeeper/pkg/constants/constants.go @@ -23,3 +23,10 @@ const ( // LabelMaster defines the label associated with master node. LabelMaster = "node-role.kubernetes.io/master" ) + +// socket file +const ( + SockDir = "/run/housekeeper-daemon" + SockName = "housekeeper-daemon.sock" +) + -- Gitee From 931e94fdae2e151e82baa38712299e977bba02f5 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Wed, 21 Jun 2023 10:13:09 +0800 Subject: [PATCH 07/38] =?UTF-8?q?=E6=94=B9=E8=BF=9Bterraform=20apply?= =?UTF-8?q?=E5=87=BD=E6=95=B0tfApply,=E5=A2=9E=E5=8A=A0tfInit=E9=98=B6?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/infra/terraform/terraform.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index 1381d82..6a5409a 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -58,8 +58,13 @@ func tfInit(dir string, platform string, target string, terraformDir string, pro ) } -func tfApply(tfFilePath string, terraformBinary string, extraOpts ...tfexec.ApplyOption) error { - tf, err := newTFExec(tfFilePath, terraformBinary) +// terraform apply +func tfApply(dir string, platform string, stage Stage, terraformDir string, applyOpts ...tfexec.ApplyOption) error { + if err := tfInit(dir, platform, stage.Name(), terraformDir, stage.Providers()); err != nil { + return err + } + + tf, err := newTFExec(dir, terraformDir) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") } @@ -69,7 +74,7 @@ func tfApply(tfFilePath string, terraformBinary string, extraOpts ...tfexec.Appl // tf.SetStderr() tf.SetLogger(newPrintfer()) - err = tf.Apply(context.Background(), extraOpts...) + err = tf.Apply(context.Background(), applyOpts...) return errors.Wrap(err, "failed to apply Terraform.") } -- Gitee From 420324d74fdd542ca07bad9c89ce662062ba5a37 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Wed, 21 Jun 2023 10:19:37 +0800 Subject: [PATCH 08/38] =?UTF-8?q?=E6=94=B9=E8=BF=9Bterraform=20destroy?= =?UTF-8?q?=E5=87=BD=E6=95=B0tfDestroy,=E5=A2=9E=E5=8A=A0tfInit=E9=98=B6?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/infra/terraform/terraform.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index 6a5409a..5d36b01 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -78,8 +78,13 @@ func tfApply(dir string, platform string, stage Stage, terraformDir string, appl return errors.Wrap(err, "failed to apply Terraform.") } -func tfDestroy(tfFilePath string, terraformBinary string, extraOpts ...tfexec.DestroyOption) error { - tf, err := newTFExec(tfFilePath, terraformBinary) +// terraform destroy +func tfDestroy(dir string, platform string, stage Stage, terraformDir string, destroyOpts ...tfexec.DestroyOption) error { + if err := tfInit(dir, platform, stage.Name(), terraformDir, stage.Providers()); err != nil { + return err + } + + tf, err := newTFExec(dir, terraformDir) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") } @@ -89,6 +94,8 @@ func tfDestroy(tfFilePath string, terraformBinary string, extraOpts ...tfexec.De // tf.SetStderr() tf.SetLogger(newPrintfer()) - err = tf.Destroy(context.Background(), extraOpts...) - return errors.Wrap(err, "failed to apply Terraform.") + return errors.Wrap( + tf.Destroy(context.Background(), destroyOpts...), + "failed doing terraform destroy.", + ) } -- Gitee From c8202a962973c22c2007b56c861734d5a320247b Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 21 Jun 2023 10:37:56 +0800 Subject: [PATCH 09/38] add controller-manager source code for housekeeper --- .../controllers/update_controller.go | 82 ++++++++++++++++++ .../operator/housekeeper-controller/main.go | 83 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 housekeeper/operator/housekeeper-controller/controllers/update_controller.go create mode 100644 housekeeper/operator/housekeeper-controller/main.go diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go new file mode 100644 index 0000000..d491836 --- /dev/null +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "os" + + "github.com/sirupsen/logrus" + housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" + "housekeeper.io/pkg/common" + "housekeeper.io/pkg/connection" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// UpdateReconciler reconciles a Update object +type UpdateReconciler struct { + client.Client + Scheme *runtime.Scheme + KubeClientSet kubernetes.Interface + Connection *connection.Client + HostName string +} + +//+kubebuilder:rbac:groups=housekeeper.io,resources=updates,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=housekeeper.io,resources=updates/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=housekeeper.io,resources=updates/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Update object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile +func NewUpdateReconciler(mgr manager.Manager) *UpdateReconciler { + kubeClientSet, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + logrus.Errorf("failed to build the kubernetes clientset: %v", err) + } + reconciler := &UpdateReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + KubeClientSet: kubeClientSet, + HostName: os.Getenv("NODE_NAME"), + } + return reconciler +} + +func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + return common.RequeueAfter, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&housekeeperiov1alpha1.Update{}). + Complete(r) +} diff --git a/housekeeper/operator/housekeeper-controller/main.go b/housekeeper/operator/housekeeper-controller/main.go new file mode 100644 index 0000000..dec60a6 --- /dev/null +++ b/housekeeper/operator/housekeeper-controller/main.go @@ -0,0 +1,83 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "path/filepath" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + "github.com/sirupsen/logrus" + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" + "housekeeper.io/operator/housekeeper-controller/controllers" + "housekeeper.io/pkg/connection" + "housekeeper.io/pkg/constants" + "housekeeper.io/pkg/version" + //+kubebuilder:scaffold:imports +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(housekeeperiov1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var err error + opts := zap.Options{} + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + HealthProbeBindAddress: "0", + }) + if err != nil { + logrus.Errorf("unable to start manager: %v", err) + os.Exit(1) + } + + reconciler := controllers.NewUpdateReconciler(mgr) + if reconciler.Connection, err = connection.New("unix://" + filepath.Join(constants.SockDir, constants.SockName)); err != nil { + logrus.Errorf("unable running housekeeper-controller: %v", err) + } + if err = reconciler.SetupWithManager(mgr); err != nil { + logrus.Error(err, "unable to create controller", "controller", "Update") + os.Exit(1) + } + + logrus.Info("starting housekeeper-controller manager version:", version.Version) + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + logrus.Errorf("problem running housekeeper-controller manager: %v", err) + os.Exit(1) + } +} -- Gitee From 8188364ecbe0a13a4f54948a6cd51730a59f0347 Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 21 Jun 2023 10:54:51 +0800 Subject: [PATCH 10/38] Add the ability for controllers to check for updates --- .../controllers/update_controller.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go index d491836..33a77fd 100644 --- a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -18,14 +18,18 @@ package controllers import ( "context" + "fmt" "os" "github.com/sirupsen/logrus" housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" "housekeeper.io/pkg/common" "housekeeper.io/pkg/connection" + "housekeeper.io/pkg/constants" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -71,9 +75,36 @@ func NewUpdateReconciler(mgr manager.Manager) *UpdateReconciler { func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) + upInstance, nodeInstance := reqInstance(ctx, r, req.NamespacedName, r.HostName) + upgradeCluster := checkUpgrade(&nodeInstance, upInstance.Spec.OSVersion, upInstance.Spec.KubeVersion) return common.RequeueAfter, nil } +func reqInstance(ctx context.Context, r common.ReadWriterClient, name types.NamespacedName, + HostName string) (upInstance housekeeperiov1alpha1.Update, nodeInstance corev1.Node) { + if err := r.Get(ctx, name, &upInstance); err != nil { + logrus.Errorf("unable to fetch update instance: %v", err) + return + } + if err := r.Get(ctx, client.ObjectKey{Name: HostName}, &nodeInstance); err != nil { + logrus.Errorf("unable to fetch node instance: %v", err) + return + } + return +} + +func checkUpgrade(node *corev1.Node, osVersionSpec string, kubeVersionSpec string) bool { + if len(kubeVersionSpec) > 0 { + labelKubeVersion := fmt.Sprintf("%s%s", constants.LabelKubeVersionPrefix, kubeVersionSpec) + if _, ok := node.Labels[labelKubeVersion]; ok { + return false + } + } else { + return node.Status.NodeInfo.OSImage != osVersionSpec + } + return true +} + // SetupWithManager sets up the controller with the Manager. func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). -- Gitee From 79b5e0a15d863d4074934f87b124a4330909b7d2 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Wed, 21 Jun 2023 11:06:02 +0800 Subject: [PATCH 11/38] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=87=BD=E6=95=B0unpac?= =?UTF-8?q?k=E4=BB=A5=E5=8F=8Aterraform=E7=9A=84=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 3 ++ go.sum | 15 +++++++++ pkg/infra/openstack.go | 20 ++++------- pkg/infra/terraform/init.go | 55 ++++++++++++++++++++++++++++++ pkg/infra/terraform/state.go | 4 +-- pkg/infra/terraform/terraform.go | 58 +++++++++++--------------------- 6 files changed, 102 insertions(+), 53 deletions(-) create mode 100644 pkg/infra/terraform/init.go diff --git a/go.mod b/go.mod index ca5e20f..a06340c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.19 require ( github.com/hashicorp/terraform-exec v0.18.1 github.com/lithammer/dedent v1.1.0 + github.com/openshift/installer v0.16.1 github.com/pkg/errors v0.9.1 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 ) @@ -17,5 +19,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/zclconf/go-cty v1.13.1 // indirect golang.org/x/crypto v0.7.0 // indirect + golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index ec697df..4738bb3 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,9 @@ github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= @@ -26,14 +28,23 @@ github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgy github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/openshift/installer v0.16.1 h1:PmjALN9x1NVNVi3SCqfz0ZwVCgOkQLQWo2nHYXREq/A= +github.com/openshift/installer v0.16.1/go.mod h1:VWGgpJgF8DGCKQjbccnigglhZnHtRLCZ6cxqkXN4Ck0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/zclconf/go-cty v1.13.1 h1:0a6bRwuiSHtAmqCqNOE+c2oHgepv0ctoxU4FUe43kwc= github.com/zclconf/go-cty v1.13.1/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= @@ -41,9 +52,13 @@ golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/infra/openstack.go b/pkg/infra/openstack.go index be3bc69..c3ad75a 100755 --- a/pkg/infra/openstack.go +++ b/pkg/infra/openstack.go @@ -22,16 +22,14 @@ import ( ) type OpenstackDeployer struct { - tfFilePath string + workingDir string + platform string + target string terraformBinary string } -func (t OpenstackDeployer) Create(extraOpts ...tfexec.ApplyOption) error { - if err := terraform.Init(t.tfFilePath, t.terraformBinary); err != nil { - return err - } - - applyErr := terraform.Apply(t.tfFilePath, t.terraformBinary, extraOpts...) +func (t OpenstackDeployer) Create(applyOpts ...tfexec.ApplyOption) error { + applyErr := terraform.TFApply(t.workingDir, t.platform, t.target, t.terraformBinary, applyOpts) if applyErr != nil { return applyErr } @@ -39,12 +37,8 @@ func (t OpenstackDeployer) Create(extraOpts ...tfexec.ApplyOption) error { return nil } -func (t OpenstackDeployer) Destroy(extraOpts ...tfexec.DestroyOption) error { - if err := terraform.Init(t.tfFilePath, t.terraformBinary); err != nil { - return err - } - - destroyErr := terraform.Destroy(t.tfFilePath, t.terraformBinary, extraOpts) +func (t OpenstackDeployer) Destroy(destroyOpts ...tfexec.DestroyOption) error { + destroyErr := terraform.TFDestroy(t.workingDir, t.platform, t.target, t.terraformBinary, destroyOpts) if destroyErr != nil { return destroyErr } diff --git a/pkg/infra/terraform/init.go b/pkg/infra/terraform/init.go new file mode 100644 index 0000000..b9d992f --- /dev/null +++ b/pkg/infra/terraform/init.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package terraform + +import ( + "context" + "path/filepath" + + "github.com/hashicorp/terraform-exec/tfexec" + "github.com/openshift/installer/data" + "github.com/pkg/errors" +) + +func unpack(workingDir string, platform string, target string) (err error) { + err = data.Unpack(workingDir, filepath.Join(platform, target)) + if err != nil { + return err + } + + return nil +} + +// terraform init +func tfInit(workingDir string, platform string, target string, terraformBinary string, providers []prov.Provider) (err error) { + err = unpack(workingDir, platform, target) + if err != nil { + return errors.Wrap(err, "failed to unpack terraform modules") + } + + tf, err := newTFExec(workingDir, terraformBinary) + if err != nil { + return errors.Wrap(err, "failed to create a new tfexec.") + } + + // 如果想导入本地已有插件，需构建相应目录 + // 如openstack插件所需目录为plugins/registry.terraform.io/terraform-provider-openstack/openstack/1.51.1/linux_arm64/terraform-provider-openstack_v1.51.1 + return errors.Wrap( + tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformBinary, "plugins"))), + "failed doing terraform init.", + ) +} diff --git a/pkg/infra/terraform/state.go b/pkg/infra/terraform/state.go index fd7481e..e1601a9 100644 --- a/pkg/infra/terraform/state.go +++ b/pkg/infra/terraform/state.go @@ -22,10 +22,10 @@ import ( "github.com/pkg/errors" ) -func Outputs(tfFilePath string, terraformBinary string) error { +func Outputs(workingDir string, terraformBinary string) error { // TODO 解析tfstate文件，读取特定内容 - tf, err := newTFExec(tfFilePath, terraformBinary) + tf, err := newTFExec(workingDir, terraformBinary) if err != nil { return err } diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index 5d36b01..c4a8922 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -18,60 +18,42 @@ package terraform import ( "context" + "os" "path/filepath" "github.com/hashicorp/terraform-exec/tfexec" "github.com/pkg/errors" ) -func newTFExec(tfFilePath string, terraformBinary string) (*tfexec.Terraform, error) { - tf, err := tfexec.NewTerraform(tfFilePath, terraformBinary) +func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, error) { + execPath := filepath.Join(terraformBinary, "bin", "terraform") + tf, err := tfexec.NewTerraform(workingDir, execPath) if err != nil { return nil, err } - // TODO 日志输出 - // tf.SetStdout() - // tf.SetStderr() - // tf.SetLogger(newPrintfer()) + // TODO 日志打印 + tf.SetStdout(os.Stdout) + tf.SetStderr(os.Stderr) + tf.SetLogger(newPrintfer()) return tf, nil } -// terraform init -func tfInit(dir string, platform string, target string, terraformDir string, providers []prov.Provider) (err error) { - err = unpack(dir, platform, target) - if err != nil { - return errors.Wrap(err, "failed to unpack Terraform modules") - } - - tf, err := newTFExec(dir, terraformDir) - if err != nil { - return errors.Wrap(err, "failed to create a new tfexec.") - } - - // 如果想导入本地已有插件，需手动构建相应目录 - // 如openstack插件所需目录为plugins/registry.terraform.io/terraform-provider-openstack/openstack/1.51.1/linux_arm64/terraform-provider-openstack_v1.51.1 - return errors.Wrap( - tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformDir, "plugins"))), - "failed doing terraform init.", - ) -} - // terraform apply -func tfApply(dir string, platform string, stage Stage, terraformDir string, applyOpts ...tfexec.ApplyOption) error { - if err := tfInit(dir, platform, stage.Name(), terraformDir, stage.Providers()); err != nil { +func TFApply(workingDir string, platform string, stage Stage, terraformBinary string, applyOpts ...tfexec.ApplyOption) error { + if err := tfInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { return err } - tf, err := newTFExec(dir, terraformDir) + tf, err := newTFExec(workingDir, terraformBinary) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") } - // TODO 日志输出 - // tf.SetStdout() - // tf.SetStderr() + // TODO 日志打印 + tf.SetStdout(os.Stdout) + tf.SetStderr(os.Stderr) tf.SetLogger(newPrintfer()) err = tf.Apply(context.Background(), applyOpts...) @@ -79,19 +61,19 @@ func tfApply(dir string, platform string, stage Stage, terraformDir string, appl } // terraform destroy -func tfDestroy(dir string, platform string, stage Stage, terraformDir string, destroyOpts ...tfexec.DestroyOption) error { - if err := tfInit(dir, platform, stage.Name(), terraformDir, stage.Providers()); err != nil { +func TFDestroy(workingDir string, platform string, stage Stage, terraformBinary string, destroyOpts ...tfexec.DestroyOption) error { + if err := tfInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { return err } - tf, err := newTFExec(dir, terraformDir) + tf, err := newTFExec(workingDir, terraformBinary) if err != nil { return errors.Wrap(err, "failed to create a new tfexec.") } - // TODO 日志输出 - // tf.SetStdout() - // tf.SetStderr() + // TODO 日志打印 + tf.SetStdout(os.Stdout) + tf.SetStderr(os.Stderr) tf.SetLogger(newPrintfer()) return errors.Wrap( -- Gitee From bc528dbdefb3fa85ada2b8ad037937191a52cba6 Mon Sep 17 00:00:00 2001 From: lauk Date: Mon, 26 Jun 2023 09:26:05 +0800 Subject: [PATCH 12/38] Add the controller implementation code --- .../controllers/update_controller.go | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go index 33a77fd..5fcf92f 100644 --- a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -75,11 +75,60 @@ func NewUpdateReconciler(mgr manager.Manager) *UpdateReconciler { func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) + ctx = context.Background() upInstance, nodeInstance := reqInstance(ctx, r, req.NamespacedName, r.HostName) upgradeCluster := checkUpgrade(&nodeInstance, upInstance.Spec.OSVersion, upInstance.Spec.KubeVersion) + if upgradeCluster { + if err := r.upgradeNodes(ctx, &upInstance, &nodeInstance); err != nil { + return common.RequeueNow, err + } + } else { + r.refreshNodes(ctx, &upInstance, &nodeInstance) + } return common.RequeueAfter, nil } +func (r *UpdateReconciler) upgradeNodes(ctx context.Context, upInstance *housekeeperiov1alpha1.Update, + node *corev1.Node) error { + controlPlane := false + if _, ok := node.Labels[constants.LabelMaster]; ok { + controlPlane = true + } + if _, ok := node.Labels[constants.LabelUpgrading]; ok { + //todo drain + pushInfo := &connection.PushInfo{ + KubeVersion: upInstance.Spec.KubeVersion, + OSImageURL: upInstance.Spec.OSImageURL, + OSVersion: upInstance.Spec.OSVersion, + ControlPlane: controlPlane, + } + if err := r.Connection.UpgradeKubeSpec(pushInfo); err != nil { + return err + } + } + + return nil +} + +func (r *UpdateReconciler) refreshNodes(ctx context.Context, upInstance *housekeeperiov1alpha1.Update, node *corev1.Node) error { + deleteLabel(ctx, r, node) + if node.Spec.Unschedulable { + //todo drain + } + return nil +} + +func deleteLabel(ctx context.Context, r common.ReadWriterClient, node *corev1.Node) error { + if _, ok := node.Labels[constants.LabelUpgrading]; ok { + delete(node.Labels, constants.LabelUpgrading) + if err := r.Update(ctx, node); err != nil { + logrus.Errorf("unable to delete %s node label: %w", node.Name, err) + return err + } + } + return nil +} + func reqInstance(ctx context.Context, r common.ReadWriterClient, name types.NamespacedName, HostName string) (upInstance housekeeperiov1alpha1.Update, nodeInstance corev1.Node) { if err := r.Get(ctx, name, &upInstance); err != nil { -- Gitee From e4eae9a68bc76171a49f21c18097c06294238bfd Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Mon, 26 Jun 2023 13:51:52 +0800 Subject: [PATCH 13/38] Add stage.go, providers.go files and modify terraform log prints --- pkg/infra/terraform/init.go | 3 +- pkg/infra/terraform/logger.go | 12 ++++++- pkg/infra/terraform/providers/providers.go | 35 ++++++++++++++++++++ pkg/infra/terraform/stage.go | 32 ++++++++++++++++++ pkg/infra/terraform/terraform.go | 38 +++++++++++++--------- 5 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 pkg/infra/terraform/providers/providers.go create mode 100644 pkg/infra/terraform/stage.go diff --git a/pkg/infra/terraform/init.go b/pkg/infra/terraform/init.go index b9d992f..e52a13b 100644 --- a/pkg/infra/terraform/init.go +++ b/pkg/infra/terraform/init.go @@ -20,6 +20,7 @@ import ( "context" "path/filepath" + prov "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/providers" "github.com/hashicorp/terraform-exec/tfexec" "github.com/openshift/installer/data" "github.com/pkg/errors" @@ -35,7 +36,7 @@ func unpack(workingDir string, platform string, target string) (err error) { } // terraform init -func tfInit(workingDir string, platform string, target string, terraformBinary string, providers []prov.Provider) (err error) { +func TFInit(workingDir string, platform string, target string, terraformBinary string, providers []prov.Provider) (err error) { err = unpack(workingDir, platform, target) if err != nil { return errors.Wrap(err, "failed to unpack terraform modules") diff --git a/pkg/infra/terraform/logger.go b/pkg/infra/terraform/logger.go index bfb609d..54add90 100644 --- a/pkg/infra/terraform/logger.go +++ b/pkg/infra/terraform/logger.go @@ -16,12 +16,22 @@ limitations under the License. package terraform +import ( + "github.com/sirupsen/logrus" +) + type printfer struct { + logger *logrus.Logger + level logrus.Level } func newPrintfer() *printfer { - return &printfer{} + return &printfer{ + logger: logrus.StandardLogger(), + level: logrus.DebugLevel, + } } func (p *printfer) Printf(format string, ifs ...interface{}) { + p.logger.Logf(p.level, format, ifs...) } diff --git a/pkg/infra/terraform/providers/providers.go b/pkg/infra/terraform/providers/providers.go new file mode 100644 index 0000000..986ee8a --- /dev/null +++ b/pkg/infra/terraform/providers/providers.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import "fmt" + +var ( + OpenStack = provider("openstack") +) + +type Provider struct { + Name string + Source string +} + +func provider(name string) Provider { + return Provider{ + Name: name, + Source: fmt.Sprintf("nkd/local/%s", name), + } +} diff --git a/pkg/infra/terraform/stage.go b/pkg/infra/terraform/stage.go new file mode 100644 index 0000000..1fbe53d --- /dev/null +++ b/pkg/infra/terraform/stage.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package terraform + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/providers" + +type Stage interface { + Name() string + + // terraform state file + StateFilename() string + + // outputs file + OutputsFilename() string + + // the list of providers that are used for the stage + Providers() []providers.Provider +} diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index c4a8922..e961f8d 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -22,7 +22,9 @@ import ( "path/filepath" "github.com/hashicorp/terraform-exec/tfexec" + "github.com/openshift/installer/pkg/lineprinter" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, error) { @@ -32,9 +34,25 @@ func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, er return nil, err } - // TODO 日志打印 - tf.SetStdout(os.Stdout) - tf.SetStderr(os.Stderr) + // If the log path is not set, terraform will not receive debug logs. + if path, ok := os.LookupEnv("TEREFORM_LOG_PATH"); ok { + if err := tf.SetLog(os.Getenv("TEREFORM_LOG")); err != nil { + logrus.Infof("Skipping setting terraform log levels: %v", err) + } else { + tf.SetLogCore(os.Getenv("TEREFORM_LOG_CORE")) //nolint:errcheck + tf.SetLogProvider(os.Getenv("TEREFORM_LOG_PROVIDER")) //nolint:errcheck + tf.SetLogPath(path) //nolint:errcheck + } + } + + // Add terraform info logs to the installer log + lpPrint := &lineprinter.LinePrinter{Print: (&lineprinter.Trimmer{WrappedPrint: logrus.Print}).Print} + lpError := &lineprinter.LinePrinter{Print: (&lineprinter.Trimmer{WrappedPrint: logrus.Error}).Print} + defer lpPrint.Close() + defer lpError.Close() + + tf.SetStdout(lpPrint) + tf.SetStderr(lpError) tf.SetLogger(newPrintfer()) return tf, nil @@ -42,7 +60,7 @@ func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, er // terraform apply func TFApply(workingDir string, platform string, stage Stage, terraformBinary string, applyOpts ...tfexec.ApplyOption) error { - if err := tfInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { + if err := TFInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { return err } @@ -51,18 +69,13 @@ func TFApply(workingDir string, platform string, stage Stage, terraformBinary st return errors.Wrap(err, "failed to create a new tfexec.") } - // TODO 日志打印 - tf.SetStdout(os.Stdout) - tf.SetStderr(os.Stderr) - tf.SetLogger(newPrintfer()) - err = tf.Apply(context.Background(), applyOpts...) return errors.Wrap(err, "failed to apply Terraform.") } // terraform destroy func TFDestroy(workingDir string, platform string, stage Stage, terraformBinary string, destroyOpts ...tfexec.DestroyOption) error { - if err := tfInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { + if err := TFInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { return err } @@ -71,11 +84,6 @@ func TFDestroy(workingDir string, platform string, stage Stage, terraformBinary return errors.Wrap(err, "failed to create a new tfexec.") } - // TODO 日志打印 - tf.SetStdout(os.Stdout) - tf.SetStderr(os.Stderr) - tf.SetLogger(newPrintfer()) - return errors.Wrap( tf.Destroy(context.Background(), destroyOpts...), "failed doing terraform destroy.", -- Gitee From cd7d0193a38bd59112533c999a952501af2bb3ba Mon Sep 17 00:00:00 2001 From: lauk Date: Mon, 26 Jun 2023 16:39:26 +0800 Subject: [PATCH 14/38] Add the ability to drain --- .../controllers/update_controller.go | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go index 5fcf92f..67feb11 100644 --- a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/kubectl/pkg/drain" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -95,7 +96,16 @@ func (r *UpdateReconciler) upgradeNodes(ctx context.Context, upInstance *houseke controlPlane = true } if _, ok := node.Labels[constants.LabelUpgrading]; ok { - //todo drain + drainer := &drain.Helper{ + Ctx: ctx, + Client: r.KubeClientSet, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, + } + if err := drainNode(drainer, node); err != nil { + return err + } pushInfo := &connection.PushInfo{ KubeVersion: upInstance.Spec.KubeVersion, OSImageURL: upInstance.Spec.OSImageURL, @@ -113,7 +123,17 @@ func (r *UpdateReconciler) upgradeNodes(ctx context.Context, upInstance *houseke func (r *UpdateReconciler) refreshNodes(ctx context.Context, upInstance *housekeeperiov1alpha1.Update, node *corev1.Node) error { deleteLabel(ctx, r, node) if node.Spec.Unschedulable { - //todo drain + drainer := &drain.Helper{ + Ctx: ctx, + Client: r.KubeClientSet, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, + } + if err := drain.cordonOrUncordonNode(false, drainer, node); err != nil { + return err + } + logrus.Infof("uncordon successfully %s node", node.Name) } return nil } @@ -129,6 +149,36 @@ func deleteLabel(ctx context.Context, r common.ReadWriterClient, node *corev1.No return nil } +func cordonOrUncordonNode(desired bool, drainer *drain.Helper, node *corev1.Node) error { + carry := "cordon" + if !desired { + carry = "uncordon" + } + logrus.Info(node.Name, "initiating %s", carry) + if node.Spec.Unschedulable == desired { + return nil + } + err := drain.RunCordonOrUncordon(drainer, node, desired) + if err != nil { + return fmt.Errorf("failed to %s: %w", carry, err) + } + return nil +} + +func drainNode(drainer *drain.Helper, node *corev1.Node) error { + logrus.Info(node.Name, "is cordoning") + // Perform cordon + if err := cordonOrUncordonNode(true, drainer, node); err != nil { + return fmt.Errorf("failed to cordon node %s: %v", node.Name, err) + } + // Attempt drain + logrus.Info(node.Name, "initiating drain") + if err := drain.RunNodeDrain(drainer, node.Name); err != nil { + return fmt.Errorf("unable to drain: %v", err) + } + return nil +} + func reqInstance(ctx context.Context, r common.ReadWriterClient, name types.NamespacedName, HostName string) (upInstance housekeeperiov1alpha1.Update, nodeInstance corev1.Node) { if err := r.Get(ctx, name, &upInstance); err != nil { -- Gitee From 2247fcf072070f838d469aa5a25ac604458d52d9 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Mon, 26 Jun 2023 17:18:49 +0800 Subject: [PATCH 15/38] add function applyTerraform --- pkg/infra/{ => assets}/assets.go | 7 +++- pkg/infra/cluster.go | 62 ++++++++++++++++++++++++++++++++ pkg/infra/infradeployer.go | 4 --- pkg/infra/openstack.go | 22 ++---------- pkg/infra/terraform/state.go | 17 +++++---- 5 files changed, 81 insertions(+), 31 deletions(-) rename pkg/infra/{ => assets}/assets.go (91%) create mode 100644 pkg/infra/cluster.go diff --git a/pkg/infra/assets.go b/pkg/infra/assets/assets.go similarity index 91% rename from pkg/infra/assets.go rename to pkg/infra/assets/assets.go index 176dd31..75fdc8d 100755 --- a/pkg/infra/assets.go +++ b/pkg/infra/assets/assets.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package infra +package assets // path : contents type Assets map[string][]byte @@ -30,3 +30,8 @@ func (a *Assets) Merge(b Assets) *Assets { type AssetsGenerator interface { GenerateAssets() Assets } + +type File struct { + Filename string + Data []byte +} diff --git a/pkg/infra/cluster.go b/pkg/infra/cluster.go new file mode 100644 index 0000000..09b6274 --- /dev/null +++ b/pkg/infra/cluster.go @@ -0,0 +1,62 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package infra + +import ( + "os" + "path/filepath" + + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/assets" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" + "github.com/hashicorp/terraform-exec/tfexec" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type Cluster struct { + FileList []*assets.File +} + +func (c *Cluster) applyTerraform(tmpDir string, platform string, stage terraform.Stage, terraformBinary string, applyOpts ...tfexec.ApplyOption) (*assets.File, error) { + applyErr := terraform.TFApply(tmpDir, platform, stage, terraformBinary, applyOpts...) + + if data, err := os.ReadFile(filepath.Join(tmpDir, terraform.StateFilename)); err == nil { + c.FileList = append(c.FileList, &assets.File{ + Filename: stage.StateFilename(), + Data: data, + }) + } else if !os.IsNotExist(err) { + logrus.Errorf("Failed to read tfstate: %v", err) + return nil, errors.Wrap(err, "failed to read tfstate") + } + + if applyErr != nil { + return nil, errors.WithMessage(applyErr, "failed to apply Terraform.") + } + + outputs, err := terraform.Outputs(tmpDir, terraformBinary) + if err != nil { + return nil, errors.Wrapf(err, "could not get outputs from stage %q", stage.Name()) + } + + outputsFile := &assets.File{ + Filename: stage.OutputsFilename(), + Data: outputs, + } + + return outputsFile, nil +} diff --git a/pkg/infra/infradeployer.go b/pkg/infra/infradeployer.go index f2c42b4..dc28150 100755 --- a/pkg/infra/infradeployer.go +++ b/pkg/infra/infradeployer.go @@ -16,8 +16,6 @@ limitations under the License. package infra -import "github.com/hashicorp/terraform-exec/tfexec" - type InfraSpec struct { diskSize string memorySize string @@ -27,6 +25,4 @@ type InfraSpec struct { } type InfraDeployer interface { - Create(spec InfraSpec, config InitConfig, extraOpts ...tfexec.ApplyOption) error - Destroy(spec InfraSpec, config InitConfig, extraOpts ...tfexec.DestroyOption) error } diff --git a/pkg/infra/openstack.go b/pkg/infra/openstack.go index c3ad75a..76d4232 100755 --- a/pkg/infra/openstack.go +++ b/pkg/infra/openstack.go @@ -16,32 +16,14 @@ limitations under the License. package infra -import ( - "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" - "github.com/hashicorp/terraform-exec/tfexec" -) - type OpenstackDeployer struct { - workingDir string - platform string - target string - terraformBinary string } -func (t OpenstackDeployer) Create(applyOpts ...tfexec.ApplyOption) error { - applyErr := terraform.TFApply(t.workingDir, t.platform, t.target, t.terraformBinary, applyOpts) - if applyErr != nil { - return applyErr - } - +func (t OpenstackDeployer) Create() error { return nil } -func (t OpenstackDeployer) Destroy(destroyOpts ...tfexec.DestroyOption) error { - destroyErr := terraform.TFDestroy(t.workingDir, t.platform, t.target, t.terraformBinary, destroyOpts) - if destroyErr != nil { - return destroyErr - } +func (t OpenstackDeployer) Destroy() error { return nil } diff --git a/pkg/infra/terraform/state.go b/pkg/infra/terraform/state.go index e1601a9..9509abc 100644 --- a/pkg/infra/terraform/state.go +++ b/pkg/infra/terraform/state.go @@ -18,21 +18,23 @@ package terraform import ( "context" + "encoding/json" "github.com/pkg/errors" ) -func Outputs(workingDir string, terraformBinary string) error { - // TODO 解析tfstate文件，读取特定内容 +const StateFilename = "terraform.tfstate" +// Reads the terraform state file. +func Outputs(workingDir string, terraformBinary string) ([]byte, error) { tf, err := newTFExec(workingDir, terraformBinary) if err != nil { - return err + return nil, err } tfoutput, err := tf.Output(context.Background()) if err != nil { - return errors.Wrap(err, "failed to read terraform state file") + return nil, errors.Wrap(err, "failed to read terraform state file") } outputs := make(map[string]interface{}, len(tfoutput)) @@ -40,7 +42,10 @@ func Outputs(workingDir string, terraformBinary string) error { outputs[key] = value.Value } - // TODO 解析outputs + data, err := json.Marshal(outputs) + if err != nil { + return nil, errors.Wrap(err, "could not marshal outputs") + } - return nil + return data, nil } -- Gitee From 5e4ed401309558ae597e6fa99ef8c7de2addb24c Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 28 Jun 2023 09:49:10 +0800 Subject: [PATCH 16/38] Add communication function between daemon and operator --- housekeeper/daemon/main.go | 33 ++++++++++++ housekeeper/daemon/server/listener.go | 75 +++++++++++++++++++++++++++ housekeeper/daemon/server/server.go | 33 ++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 housekeeper/daemon/main.go create mode 100644 housekeeper/daemon/server/listener.go create mode 100644 housekeeper/daemon/server/server.go diff --git a/housekeeper/daemon/main.go b/housekeeper/daemon/main.go new file mode 100644 index 0000000..c843375 --- /dev/null +++ b/housekeeper/daemon/main.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "os" + + "github.com/sirupsen/logrus" + "housekeeper.io/daemon/server" + "housekeeper.io/pkg/version" + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +func main() { + logrus.Info("Version is:", version.Version) + if err := server.Run(); err != nil { + logrus.Errorln("listen error" + err.Error()) + os.Exit(1) + } +} diff --git a/housekeeper/daemon/server/listener.go b/housekeeper/daemon/server/listener.go new file mode 100644 index 0000000..007ef7f --- /dev/null +++ b/housekeeper/daemon/server/listener.go @@ -0,0 +1,75 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package server + +import ( + "fmt" + "net" + "os" + "path/filepath" + "syscall" + + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + pb "housekeeper.io/pkg/connection/proto" + "housekeeper.io/pkg/constants" +) + +func NewListener(dir, name string) (l net.Listener, err error) { + if err := os.MkdirAll(dir, 0750); err != nil { + return nil, err + } + + addr := filepath.Join(dir, name) + gid := os.Getgid() + if err = syscall.Unlink(addr); err != nil && !os.IsNotExist(err) { + return nil, err + } + + const socketPermission = 0640 + mask := syscall.Umask(^socketPermission & int(os.ModePerm)) + defer syscall.Umask(mask) + + l, err = net.Listen("unix", addr) + if err != nil { + return nil, err + } + + if err := os.Chown(addr, 0, gid); err != nil { + if err := l.Close(); err != nil { + return nil, fmt.Errorf("close listener error %w", err) + } + return nil, err + } + return l, nil +} + +func Run() error { + lis, err := NewListener(constants.SockDir, constants.SockName) + if err != nil { + logrus.Errorf("listen error: %v", err) + return err + } + //get grpc server + s := grpc.NewServer() + pb.RegisterUpgradeClusterServer(s, &Server{}) + logrus.Info("housekeeper-daemon start serving") + if err := s.Serve(lis); err != nil { + logrus.Errorf("housekeeper-daemon server error: %v", err) + return err + } + return nil +} diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go new file mode 100644 index 0000000..42a5b85 --- /dev/null +++ b/housekeeper/daemon/server/server.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + + pb "housekeeper.io/pkg/connection/proto" +) + +type Server struct { + pb.UnimplementedUpgradeClusterServer +} + +// Implements the Upgrade +func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.UpgradeResponse, error) { + //todo + return &pb.UpgradeResponse{}, nil +} -- Gitee From e3bb1d5cd04d1037a0e3703eb4fd860a8a05291c Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Thu, 29 Jun 2023 09:30:53 +0800 Subject: [PATCH 17/38] =?UTF-8?q?infra=E6=A8=A1=E5=9D=97=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=87=BD=E6=95=B0Create()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/infra/{ => assets/cluster}/cluster.go | 58 ++++++++++++++++++- pkg/infra/assets/cluster/tfvars.go | 27 +++++++++ .../terraform/stages/openstack/stages.go | 19 ++++++ pkg/infra/terraform/stages/platform/stages.go | 19 ++++++ pkg/infra/terraform/stages/split.go | 43 ++++++++++++++ pkg/infra/terraform/terraform.go | 4 +- 6 files changed, 167 insertions(+), 3 deletions(-) rename pkg/infra/{ => assets/cluster}/cluster.go (47%) create mode 100644 pkg/infra/assets/cluster/tfvars.go create mode 100644 pkg/infra/terraform/stages/openstack/stages.go create mode 100644 pkg/infra/terraform/stages/platform/stages.go create mode 100644 pkg/infra/terraform/stages/split.go diff --git a/pkg/infra/cluster.go b/pkg/infra/assets/cluster/cluster.go similarity index 47% rename from pkg/infra/cluster.go rename to pkg/infra/assets/cluster/cluster.go index 09b6274..c22963e 100644 --- a/pkg/infra/cluster.go +++ b/pkg/infra/assets/cluster/cluster.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package infra +package cluster import ( "os" @@ -22,6 +22,7 @@ import ( "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/assets" "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" + platformStages "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/stages/platform" "github.com/hashicorp/terraform-exec/tfexec" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -31,6 +32,61 @@ type Cluster struct { FileList []*assets.File } +func (c *Cluster) Create(platform string, node string, installDir string) (err error) { + if installDir == "" { + logrus.Fatal("InstallDir has not been set") + } + + terraformVariables := &TerraformVariables{} + stages, err := platformStages.StagesForPlatform(platform, node) + + terraformBinary := filepath.Join(installDir, "terraform") + _, err = os.Stat(terraformBinary) + if os.IsNotExist(err) { + if err := os.Mkdir(terraformBinary, 0777); err != nil { + return errors.Wrap(err, "could not create the terraform directory") + } + } + + // TODO 将所需二进制文件保存到terraformBinary目录 + + terraformBinaryPath, err := filepath.Abs(terraformBinary) + if err != nil { + return errors.Wrap(err, "cannot get absolute path of terraform directory") + } + + tfvarsFiles := make([]*assets.File, 0, len(terraformVariables.Files())+len(stages)) + tfvarsFiles = append(tfvarsFiles, terraformVariables.Files()...) + + for _, stage := range stages { + outputs, err := c.applyStage(platform, stage, terraformBinaryPath, tfvarsFiles) + if err != nil { + return errors.Wrapf(err, "failure applying terraform for %q stage", stage.Name()) + } + tfvarsFiles = append(tfvarsFiles, outputs) + c.FileList = append(c.FileList, outputs) + } + + return nil +} + +func (c *Cluster) applyStage(platform string, stage terraform.Stage, terraformBinary string, tfvarsFiles []*assets.File) (*assets.File, error) { + tmpDir := filepath.Join(terraformBinary, platform, stage.Name()) + if err := os.MkdirAll(tmpDir, 0777); err != nil { + return nil, errors.Wrapf(err, "cannot create the directory for %s", stage.Name()) + } + + var applyOpts []tfexec.ApplyOption + for _, file := range tfvarsFiles { + if err := os.WriteFile(filepath.Join(tmpDir, file.Filename), file.Data, 0o600); err != nil { + return nil, err + } + applyOpts = append(applyOpts, tfexec.VarFile(filepath.Join(tmpDir, file.Filename))) + } + + return c.applyTerraform(tmpDir, platform, stage, terraformBinary, applyOpts...) +} + func (c *Cluster) applyTerraform(tmpDir string, platform string, stage terraform.Stage, terraformBinary string, applyOpts ...tfexec.ApplyOption) (*assets.File, error) { applyErr := terraform.TFApply(tmpDir, platform, stage, terraformBinary, applyOpts...) diff --git a/pkg/infra/assets/cluster/tfvars.go b/pkg/infra/assets/cluster/tfvars.go new file mode 100644 index 0000000..6d5729f --- /dev/null +++ b/pkg/infra/assets/cluster/tfvars.go @@ -0,0 +1,27 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/assets" + +type TerraformVariables struct { + FileList []*assets.File +} + +func (t *TerraformVariables) Files() []*assets.File { + return t.FileList +} diff --git a/pkg/infra/terraform/stages/openstack/stages.go b/pkg/infra/terraform/stages/openstack/stages.go new file mode 100644 index 0000000..93ad003 --- /dev/null +++ b/pkg/infra/terraform/stages/openstack/stages.go @@ -0,0 +1,19 @@ +package openstack + +import ( + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/providers" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/stages" +) + +var PlatformStages = []terraform.Stage{} + +func AddPlatformStage(stage string) { + newStage := stages.NewStage( + "openstack", + stage, + []providers.Provider{providers.OpenStack}, + ) + + PlatformStages = append(PlatformStages, newStage) +} diff --git a/pkg/infra/terraform/stages/platform/stages.go b/pkg/infra/terraform/stages/platform/stages.go new file mode 100644 index 0000000..7a99cad --- /dev/null +++ b/pkg/infra/terraform/stages/platform/stages.go @@ -0,0 +1,19 @@ +package platform + +import ( + "errors" + "fmt" + + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/stages/openstack" +) + +func StagesForPlatform(platform string, stage string) ([]terraform.Stage, error) { + switch platform { + case "openstack": + openstack.AddPlatformStage(stage) + return openstack.PlatformStages, nil + default: + return nil, errors.New(fmt.Sprintf("unsupported platform %q", platform)) + } +} diff --git a/pkg/infra/terraform/stages/split.go b/pkg/infra/terraform/stages/split.go new file mode 100644 index 0000000..caf0de7 --- /dev/null +++ b/pkg/infra/terraform/stages/split.go @@ -0,0 +1,43 @@ +package stages + +import ( + "fmt" + + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/providers" +) + +type StageOption func(*SplitStage) + +func NewStage(platform, name string, providers []providers.Provider, opts ...StageOption) SplitStage { + s := SplitStage{ + platform: platform, + name: name, + providers: providers, + } + for _, opt := range opts { + opt(&s) + } + return s +} + +type SplitStage struct { + platform string + name string + providers []providers.Provider +} + +func (s SplitStage) Name() string { + return s.name +} + +func (s SplitStage) Providers() []providers.Provider { + return s.providers +} + +func (s SplitStage) StateFilename() string { + return fmt.Sprintf("terraform.%s.tfstate", s.name) +} + +func (s SplitStage) OutputsFilename() string { + return fmt.Sprintf("%s.tfvars.json", s.name) +} diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index e961f8d..ba540da 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -35,13 +35,13 @@ func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, er } // If the log path is not set, terraform will not receive debug logs. - if path, ok := os.LookupEnv("TEREFORM_LOG_PATH"); ok { + if logPath, ok := os.LookupEnv("TEREFORM_LOG_PATH"); ok { if err := tf.SetLog(os.Getenv("TEREFORM_LOG")); err != nil { logrus.Infof("Skipping setting terraform log levels: %v", err) } else { tf.SetLogCore(os.Getenv("TEREFORM_LOG_CORE")) //nolint:errcheck tf.SetLogProvider(os.Getenv("TEREFORM_LOG_PROVIDER")) //nolint:errcheck - tf.SetLogPath(path) //nolint:errcheck + tf.SetLogPath(logPath) //nolint:errcheck } } -- Gitee From 087452f0ebe479671771f54e6e5c4ed86c3b4e73 Mon Sep 17 00:00:00 2001 From: lauk Date: Thu, 29 Jun 2023 14:05:12 +0800 Subject: [PATCH 18/38] Add code to implement os upgrade --- housekeeper/daemon/server/server.go | 48 ++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go index 42a5b85..1cd4637 100644 --- a/housekeeper/daemon/server/server.go +++ b/housekeeper/daemon/server/server.go @@ -17,17 +17,63 @@ limitations under the License. package server import ( + "bytes" "context" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "github.com/sirupsen/logrus" pb "housekeeper.io/pkg/connection/proto" ) +const ostreeImage = "ostree-unverified-image:docker://" + type Server struct { pb.UnimplementedUpgradeClusterServer + mu sync.Mutex } // Implements the Upgrade func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.UpgradeResponse, error) { - //todo + s.mu.Lock() + defer s.mu.Unlock() + + if len(req.OsVersion) > 0 { + if err := upgradeOSVersion(req); err != nil { + logrus.Errorf("upgrade os version error: %v", err) + return &pb.UpgradeResponse{}, err + } + } return &pb.UpgradeResponse{}, nil } + +func upgradeOSVersion(req *pb.UpgradeRequest) error { + //upgrade os + customImageURL := fmt.Sprintf("%s%s", ostreeImage, req.OsImageUrl) + args := []string{"rebase", "--experimental", customImageURL, "--bypass-driver"} + if err := runCmd("rpm-ostree", args...); err != nil { + logrus.Errorf("failed to upgrade os to %s : %w", req.OsVersion, err) + return err + } + // todo：skipping reboot + rebootArgs := []string{"-c", "systemctl reboot"} + if err := runCmd("/bin/sh", rebootArgs...); err != nil { + logrus.Errorf("failed to run reboot: %v", err) + return err + } + return nil +} + +func runCmd(name string, args ...string) error { + cmd := exec.Command(name, args...) + var stderr bytes.Buffer + cmd.Stdout = os.Stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("error running %s %s: %s: %w", name, strings.Join(args, " "), string(stderr.Bytes()), err) + } + return nil +} -- Gitee From a0bebfdd5e07d840325613e6a723b06f46403f8b Mon Sep 17 00:00:00 2001 From: lauk Date: Mon, 3 Jul 2023 09:06:54 +0800 Subject: [PATCH 19/38] Add code to implement k8s upgrade --- housekeeper/daemon/server/server.go | 128 ++++++++++++++++++++++++---- 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go index 1cd4637..b4eab9b 100644 --- a/housekeeper/daemon/server/server.go +++ b/housekeeper/daemon/server/server.go @@ -17,19 +17,24 @@ limitations under the License. package server import ( - "bytes" "context" "fmt" - "os" "os/exec" "strings" "sync" "github.com/sirupsen/logrus" pb "housekeeper.io/pkg/connection/proto" + utilVersion "k8s.io/apimachinery/pkg/util/version" ) -const ostreeImage = "ostree-unverified-image:docker://" +const ( + ostreeImage = "ostree-unverified-image:docker://" + kubeadmCmd = "/usr/local/bin/kubeadm" + upgradeControlPlaneCmd = "/usr/local/bin/kubeadm upgrade apply -y" + upgradeNodesCmd = "/usr/local/bin/kubeadm upgrade node" + kubeletUpdateCmd = "systemctl daemon-reload && systemctl restart kubelet" +) type Server struct { pb.UnimplementedUpgradeClusterServer @@ -41,39 +46,130 @@ func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.Upgrade s.mu.Lock() defer s.mu.Unlock() + // upgrade os if len(req.OsVersion) > 0 { - if err := upgradeOSVersion(req); err != nil { - logrus.Errorf("upgrade os version error: %v", err) + //Checking for os version + if err := checkOsVersion(req); err != nil { + return &pb.UpgradeResponse{}, err + } + } + // upgrade kubernetes + if len(req.KubeVersion) > 0 { + //Checking for kubernetes version + if err := checkKubeVersion(req); err != nil { return &pb.UpgradeResponse{}, err } } return &pb.UpgradeResponse{}, nil } +func checkOsVersion(req *pb.UpgradeRequest) error { + args := []string{"-c", "cat /etc/os-release | grep 'VERSION=' | head -n 1 | awk -F 'VERSION=' '{print $2}'"} + osVersion, err := runCmd("/bin/sh", args...) + if err != nil { + logrus.Errorf("failed to get os version: %v", err) + return err + } + if cmp, err := utilVersion.MustParseSemantic(string(osVersion)).Compare(req.OsVersion); err != nil { + logrus.Errorf("failed to parse os version: %v", err) + return err + } else if cmp == 0 { + logrus.Infof("The current os version %s and the desired upgrade version %s are the same", + string(osVersion), req.OsVersion) + return nil + } + //Compare the current os version with the desired version. + //If different, the update command is executed + if err := upgradeOSVersion(req); err != nil { + logrus.Errorf("upgrade os version error: %v", err) + return err + } + return nil +} + +func checkKubeVersion(req *pb.UpgradeRequest) error { + args := []string{"version", "-o", "short"} + kubeadmVersion, err := runCmd(kubeadmCmd, args...) + if err != nil { + logrus.Errorf("kubeadm get version failed: %v", err) + return err + } + if cmp, err := utilVersion.MustParseSemantic(string(kubeadmVersion)).Compare(req.KubeVersion); err != nil { + logrus.Errorf("failed to parse kubeadm version: %v", err) + return err + } else if cmp == -1 { + logrus.Infof("The request upgraded version %s is larger than kubeadm's version %s", + req.KubeVersion, string(kubeadmVersion)) + return nil + } + //If the version of kubeadm is not less than the version requested for upgrade, + //the upgrade command is executed + if err := upgradeKubeVersion(req); err != nil { + logrus.Errorf("upgrade kubernetes version error: %v", err) + return err + } + return nil +} + func upgradeOSVersion(req *pb.UpgradeRequest) error { //upgrade os customImageURL := fmt.Sprintf("%s%s", ostreeImage, req.OsImageUrl) args := []string{"rebase", "--experimental", customImageURL, "--bypass-driver"} - if err := runCmd("rpm-ostree", args...); err != nil { + if _, err := runCmd("rpm-ostree", args...); err != nil { logrus.Errorf("failed to upgrade os to %s : %w", req.OsVersion, err) return err } - // todo：skipping reboot - rebootArgs := []string{"-c", "systemctl reboot"} - if err := runCmd("/bin/sh", rebootArgs...); err != nil { + // todo：skipping restart system + if err := exec.Command("/bin/sh", "-c", "systemctl reboot").Run(); err != nil { logrus.Errorf("failed to run reboot: %v", err) return err } return nil } -func runCmd(name string, args ...string) error { - cmd := exec.Command(name, args...) - var stderr bytes.Buffer - cmd.Stdout = os.Stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("error running %s %s: %s: %w", name, strings.Join(args, " "), string(stderr.Bytes()), err) +func upgradeKubeVersion(req *pb.UpgradeRequest) error { + // todo: pull images before performing the upgrade + kubeVersion := req.KubeVersion + if req.ControlPlane { + if err := upgradeControlPlaneNode(kubeVersion); err != nil { + return fmt.Errorf("failed to upgrade master nodes: %v", err) + } + } else { + if err := upgradeNodes(); err != nil { + return fmt.Errorf("failed to upgrade worker nodes: %v", err) + } + } + // todo: Mark upgrade complete + return nil +} + +func upgradeControlPlaneNode(version string) error { + args := []string{"-c", upgradeControlPlaneCmd, version} + if err := exec.Command("/bin/sh", args...).Run(); err != nil { + return fmt.Errorf("failed to upgrade nodes: %w", err) + } + if err := exec.Command("/bin/sh", "-c", kubeletUpdateCmd).Run(); err != nil { + return fmt.Errorf("failed to restart kubelet: %w", err) } return nil } + +func upgradeNodes() error { + if err := exec.Command("/bin/sh", "-c", upgradeNodesCmd).Run(); err != nil { + return fmt.Errorf("failed to upgrade nodes: %w", err) + } + if err := exec.Command("/bin/sh", "-c", kubeletUpdateCmd).Run(); err != nil { + return fmt.Errorf("failed to restart kubelet: %w", err) + } + return nil +} + +func runCmd(name string, args ...string) ([]byte, error) { + cmd := exec.Command(name, args...) + output, err := cmd.Output() + if err != nil { + logrus.Errorf("error running %s: %s: %w", name, strings.Join(args, " "), err) + return nil, err + } + return output, nil +} -- Gitee From 18b864d06b1f13a2f9a6923e3b97e8155b7a955d Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 5 Jul 2023 09:49:12 +0800 Subject: [PATCH 20/38] add k8s cluster upgrade command --- cmd/nkd/main.go | 40 ++++++++++++++++++++++++++++++---------- cmd/nkd/upgrade.go | 21 ++++++++++----------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/cmd/nkd/main.go b/cmd/nkd/main.go index 66ce32f..e4eaf8e 100755 --- a/cmd/nkd/main.go +++ b/cmd/nkd/main.go @@ -1,17 +1,37 @@ package main -import "github.com/spf13/cobra" +import ( + "os" + "path/filepath" -func addCommands(command *cobra.Command){ - command.AddCommand(generateCommand) - command.AddCommand(deployCommand) - //command.AddCommand(initCommand) - command.AddCommand(joinCommand) - command.AddCommand(upgradeCommand) -} + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) +func newRootCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: filepath.Base(os.Args[0]), + Short: "Create kubernetes cluster", + Long: "", + } + return cmd +} func main() { - addCommands(rootCmd) - _ = rootCmd.Execute() + rootCmd := newRootCmd() + + for _, subCmd := range []*cobra.Command{ + /*todo: + newDeployCmd() + newGenerateCmd() + newInitCmd() + newJoinCommand() + */ + newUpgradeCmd(), + } { + rootCmd.AddCommand(subCmd) + } + if err := rootCmd.Execute(); err != nil { + logrus.Errorf("Error executing nkd: %v", err) + } } diff --git a/cmd/nkd/upgrade.go b/cmd/nkd/upgrade.go index 087630f..a2b86b5 100755 --- a/cmd/nkd/upgrade.go +++ b/cmd/nkd/upgrade.go @@ -4,18 +4,17 @@ import ( "github.com/spf13/cobra" ) -var ( - upgradeCommand = &cobra.Command{ - Use: "deploy [nodes]", - Short: "deploy masters ", - Long: "deploy cluster bootconfig for a new cluster", - RunE: upgrade, - Args: cobra.MaximumNArgs(1), +func newUpgradeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "upgrade", + Short: "Upgrade your cluster to a newer version", + Long: "", + RunE: runUpgradeCmd, } -) - + return cmd +} -func upgrade(command *cobra.Command, args []string) error{ - println("deploy") +func runUpgradeCmd(command *cobra.Command, args []string) error { + //todo: Upgrade k8s version function implementation return nil } -- Gitee From 29d33e74fc7c21886b4355a57bb54179928bdd2a Mon Sep 17 00:00:00 2001 From: lauk Date: Tue, 11 Jul 2023 13:54:15 +0800 Subject: [PATCH 21/38] Add mark when the node is upgraded --- housekeeper/daemon/server/server.go | 78 +++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 11 deletions(-) diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go index b4eab9b..de06f32 100644 --- a/housekeeper/daemon/server/server.go +++ b/housekeeper/daemon/server/server.go @@ -19,6 +19,7 @@ package server import ( "context" "fmt" + "os" "os/exec" "strings" "sync" @@ -45,7 +46,7 @@ type Server struct { func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.UpgradeResponse, error) { s.mu.Lock() defer s.mu.Unlock() - + markFile := fmt.Sprintf("%s%s%s", "/var/housekeeper/", req.KubeVersion, ".stamp") // upgrade os if len(req.OsVersion) > 0 { //Checking for os version @@ -53,6 +54,9 @@ func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.Upgrade return &pb.UpgradeResponse{}, err } } + if isFileExist(markFile) { + return &pb.UpgradeResponse{}, nil + } // upgrade kubernetes if len(req.KubeVersion) > 0 { //Checking for kubernetes version @@ -60,6 +64,11 @@ func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.Upgrade return &pb.UpgradeResponse{}, err } } + // todo: check upgrade successfully + if err := markNode(markFile); err != nil { + logrus.Errorf("failed to mark node: %v", err) + return &pb.UpgradeResponse{}, err + } return &pb.UpgradeResponse{}, nil } @@ -130,40 +139,87 @@ func upgradeOSVersion(req *pb.UpgradeRequest) error { func upgradeKubeVersion(req *pb.UpgradeRequest) error { // todo: pull images before performing the upgrade kubeVersion := req.KubeVersion - if req.ControlPlane { - if err := upgradeControlPlaneNode(kubeVersion); err != nil { - return fmt.Errorf("failed to upgrade master nodes: %v", err) + if ok, err := isControlPlaneNode(); err != nil { + return err + } else if ok { + if err = upgradeControlPlaneNode(kubeVersion); err != nil { + logrus.Errorf("failed to upgrade master nodes: %v", err) + return err } } else { - if err := upgradeNodes(); err != nil { - return fmt.Errorf("failed to upgrade worker nodes: %v", err) + if err = upgradeNodes(); err != nil { + logrus.Errorf("failed to upgrade worker nodes: %v", err) + return err } } - // todo: Mark upgrade complete return nil } func upgradeControlPlaneNode(version string) error { args := []string{"-c", upgradeControlPlaneCmd, version} if err := exec.Command("/bin/sh", args...).Run(); err != nil { - return fmt.Errorf("failed to upgrade nodes: %w", err) + logrus.Errorf("failed to upgrade nodes: %w", err) + return err } if err := exec.Command("/bin/sh", "-c", kubeletUpdateCmd).Run(); err != nil { - return fmt.Errorf("failed to restart kubelet: %w", err) + logrus.Errorf("failed to restart kubelet: %w", err) + return err } return nil } func upgradeNodes() error { if err := exec.Command("/bin/sh", "-c", upgradeNodesCmd).Run(); err != nil { - return fmt.Errorf("failed to upgrade nodes: %w", err) + logrus.Errorf("failed to upgrade nodes: %w", err) + return err } if err := exec.Command("/bin/sh", "-c", kubeletUpdateCmd).Run(); err != nil { - return fmt.Errorf("failed to restart kubelet: %w", err) + logrus.Errorf("failed to restart kubelet: %w", err) + return err + } + return nil +} + +func isControlPlaneNode() (bool, error) { + if !isFileExist("/etc/kubernetes/admin.conf") { + return false, nil + } + ipArgs := []string{"-c", "ifconfig | grep 'inet' | grep 'broadcast'| awk '{print $2}'"} + ipAddress, err := runCmd("/bin/sh", ipArgs...) + if err != nil { + return false, err + } + adminArgs := []string{"-c", "cat /etc/kubernetes/admin.conf | grep 'server' |grep -E -o '([0-9]{1,3}.){3}[0-9]{1,3}'"} + ipControlPlane, err := runCmd("/bin/sh", adminArgs...) + if err != nil { + return false, err + } + return string(ipControlPlane) == string(ipAddress), nil +} + +func markNode(file string) error { + if err := os.MkdirAll("/var/housekeeper", 0644); err != nil { + return err + } + args := []string{"-c", "touch", file} + _, err := runCmd("/bin/sh", args...) + if err != nil { + return err } return nil } +func isFileExist(path string) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + if fileInfo.IsDir() { + return false + } + return true +} + func runCmd(name string, args ...string) ([]byte, error) { cmd := exec.Command(name, args...) output, err := cmd.Output() -- Gitee From ecd6e5233651de95f3e513f1ff72b82c52c848b2 Mon Sep 17 00:00:00 2001 From: duyiwei Date: Tue, 11 Jul 2023 14:48:00 +0800 Subject: [PATCH 22/38] Add K8S certificate management section --- pkg/cert/cacert.go | 74 +++++++++++++++++++++++++++++++++++++++++++++ pkg/cert/certapi.go | 36 ++++++++++++++++++++++ pkg/cert/certs.go | 69 ++++++++++++++++++++++++++++++++++++++++++ pkg/cert/tools.go | 67 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 pkg/cert/cacert.go create mode 100644 pkg/cert/certapi.go create mode 100644 pkg/cert/certs.go create mode 100644 pkg/cert/tools.go diff --git a/pkg/cert/cacert.go b/pkg/cert/cacert.go new file mode 100644 index 0000000..55a5c53 --- /dev/null +++ b/pkg/cert/cacert.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "time" +) + +func (cm *CertificateManager) GenerateCACertificate() error { + + // 生成CA的私钥和公钥 + var err error + cm.CAKey, err = rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + caPublicKey := &cm.CACert.PublicKey + + // 生成一个介于 0 和 2^128 - 1 之间的随机序列号，并将结果存储在 serialNumber 变量中 + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return err + } + now := time.Now() + + // 设置生成证书的参数，构建CA证书模板 + caTemplate := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"NKD"}, + CommonName: "CA", + }, //这里还可以加很多参数信息 + NotBefore: now, + NotAfter: now.AddDate(10, 0, 0), // 有效期为10年 + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, // openssl 中的 keyUsage 字段 + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, // openssl 中的 extendedKeyUsage = clientAuth, serverAuth 字段 + BasicConstraintsValid: true, + IsCA: true, //表示用于CA + } + + //caCertBytes是生成证书的中间步骤，它用于将证书的二进制表示存储在内存中，以便后续操作可以使用它 + caCertBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caPublicKey, cm.CAKey) + if err != nil { + return err + } + //cm.CACert表示已经生成的CA证书，用于存储CA证书的详细信息，例如证书序列号、主题、有效期等 + cm.CACert, err = x509.ParseCertificate(caCertBytes) + if err != nil { + return err + } + + return nil + +} diff --git a/pkg/cert/certapi.go b/pkg/cert/certapi.go new file mode 100644 index 0000000..a28444e --- /dev/null +++ b/pkg/cert/certapi.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "crypto/rsa" + "crypto/x509" +) + +type CertificateGenerator interface { + GenerateCACertificate() error + GenerateCertificate(commonName string) error +} + +// CertificateManager 是证书管理器 +type CertificateManager struct { + CAKey *rsa.PrivateKey + CACert *x509.Certificate + ComponentKey *rsa.PrivateKey + ComponentCert *x509.Certificate + ValidDays int +} diff --git a/pkg/cert/certs.go b/pkg/cert/certs.go new file mode 100644 index 0000000..838d74d --- /dev/null +++ b/pkg/cert/certs.go @@ -0,0 +1,69 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "time" +) + +// 使用CA证书和私钥生成组件证书和私钥 +func (cm *CertificateManager) GenerateComponentCertificate(componentName string) error { + + // 创建组件的公钥和私钥 + var err error + cm.ComponentKey, err = rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + componentPublicKey := &cm.ComponentKey.PublicKey + + // 生成一个介于 0 和 2^128 - 1 之间的随机序列号，并将结果存储在 serialNumber 变量中 + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return err + } + now := time.Now() + + // 组件证书模板 + componentTemplate := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"NKD"}, CommonName: componentName}, + NotBefore: now, + NotAfter: time.Now().AddDate(0, 0, cm.ValidDays), // 有效期为1年 + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + + // 使用CA证书和私钥生成组件证书 + componentCertBytes, err := x509.CreateCertificate(rand.Reader, componentTemplate, cm.CACert, componentPublicKey, cm.CAKey) + if err != nil { + return err + } + + cm.ComponentCert, err = x509.ParseCertificate(componentCertBytes) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/cert/tools.go b/pkg/cert/tools.go new file mode 100644 index 0000000..9eb5a26 --- /dev/null +++ b/pkg/cert/tools.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "crypto/x509" + "encoding/pem" + "io/ioutil" +) + +const certpath = "/etc/kubernetes/pki/" + +// CACertPEM 返回CA证书的PEM格式字节切片 +func (cm *CertificateManager) CACertPEM() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: cm.CACert.Raw, + }) +} + +// CAKeyPEM 返回CA私钥的PEM格式字节切片 +func (cm *CertificateManager) CAKeyPEM() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(cm.CAKey), + }) +} + +// ComponentCertPEM 返回组件证书的PEM格式字节切片 +func (cm *CertificateManager) ComponentCertPEM() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: cm.ComponentCert.Raw, + }) +} + +// ComponentKeyPEM 返回组件私钥的PEM格式字节切片 +func (cm *CertificateManager) ComponentKeyPEM() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(cm.ComponentKey), + }) +} + +// SaveCertificateToFile 将证书保存到文件 +func SaveCertificateToFile(filename string, certPEM []byte) error { + return ioutil.WriteFile(certpath+filename, certPEM, 0644) +} + +// SavePrivateKeyToFile 将私钥保存到文件 +func SavePrivateKeyToFile(filename string, keyPEM []byte) error { + return ioutil.WriteFile(certpath+filename, keyPEM, 0600) +} -- Gitee From d1c2c8fd5d48bb72269f8c2c72bf202479ebfab2 Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 12 Jul 2023 15:16:39 +0800 Subject: [PATCH 23/38] Change the judgment of successfully upgraded nodes --- .../controllers/update_controller.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/housekeeper/operator/housekeeper-operator/controllers/update_controller.go b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go index 1346a26..d202332 100644 --- a/housekeeper/operator/housekeeper-operator/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "github.com/sirupsen/logrus" housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" @@ -124,30 +123,34 @@ func getNodes(ctx context.Context, r common.ReadWriterClient, reqs ...labels.Req // Add the label to nodes func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []corev1.Node, name types.NamespacedName) (bool, error) { var upInstance housekeeperiov1alpha1.Update - upgradeNum := -1 if err := r.Get(ctx, name, &upInstance); err != nil { logrus.Errorf("unable to get update Instance %v", err) return false, err } var ( + upgradeNum = -1 kubeVersionSpec = upInstance.Spec.KubeVersion osVersionSpec = upInstance.Spec.OSVersion ) - //labelKubeVersion added after kube version upgrade - labelKubeVersion := fmt.Sprintf("%s%s", constants.LabelKubeVersionPrefix, kubeVersionSpec) if len(osVersionSpec) == 0 { logrus.Warning("os version is required") return false, nil } for _, node := range nodeList { + var ( + kubeProxyVersion = node.Status.NodeInfo.KubeProxyVersion + kubeletVersion = node.Status.NodeInfo.KubeletVersion + osVersion = node.Status.NodeInfo.OSImage + ) if len(kubeVersionSpec) > 0 { - if _, ok := node.Labels[labelKubeVersion]; ok { + //If kube-proxy, kubelet are the same as the version to be upgraded k8s, then k8s is successfully upgraded + if kubeVersionSpec == kubeProxyVersion && kubeVersionSpec == kubeletVersion { logrus.Infof("successfully upgraded the node: %s", node.Name) upgradeNum++ continue } } else { - if osVersionSpec == node.Status.NodeInfo.OSImage { + if osVersionSpec == osVersion { continue } } -- Gitee From c212b7e4165e88a3f162ed5a0edee3c611614aa0 Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 12 Jul 2023 15:37:50 +0800 Subject: [PATCH 24/38] Optimize the drain strategy of the controller --- housekeeper/daemon/server/server.go | 16 +---- .../controllers/update_controller.go | 60 +++++++++++-------- housekeeper/pkg/common/common.go | 12 ++++ housekeeper/pkg/connection/connection.go | 14 ++--- housekeeper/pkg/connection/proto/daemon.pb.go | 53 +++++++--------- housekeeper/pkg/connection/proto/daemon.proto | 1 - housekeeper/pkg/constants/constants.go | 3 - 7 files changed, 78 insertions(+), 81 deletions(-) diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go index de06f32..a85f234 100644 --- a/housekeeper/daemon/server/server.go +++ b/housekeeper/daemon/server/server.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/sirupsen/logrus" + "housekeeper.io/pkg/common" pb "housekeeper.io/pkg/connection/proto" utilVersion "k8s.io/apimachinery/pkg/util/version" ) @@ -54,7 +55,7 @@ func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.Upgrade return &pb.UpgradeResponse{}, err } } - if isFileExist(markFile) { + if common.IsFileExist(markFile) { return &pb.UpgradeResponse{}, nil } // upgrade kubernetes @@ -181,7 +182,7 @@ func upgradeNodes() error { } func isControlPlaneNode() (bool, error) { - if !isFileExist("/etc/kubernetes/admin.conf") { + if !common.IsFileExist("/etc/kubernetes/admin.conf") { return false, nil } ipArgs := []string{"-c", "ifconfig | grep 'inet' | grep 'broadcast'| awk '{print $2}'"} @@ -209,17 +210,6 @@ func markNode(file string) error { return nil } -func isFileExist(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - if fileInfo.IsDir() { - return false - } - return true -} - func runCmd(name string, args ...string) ([]byte, error) { cmd := exec.Command(name, args...) output, err := cmd.Output() diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go index 67feb11..6546451 100644 --- a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/kubectl/pkg/drain" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -78,7 +79,13 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr _ = log.FromContext(ctx) ctx = context.Background() upInstance, nodeInstance := reqInstance(ctx, r, req.NamespacedName, r.HostName) - upgradeCluster := checkUpgrade(&nodeInstance, upInstance.Spec.OSVersion, upInstance.Spec.KubeVersion) + var ( + osVersionSpec = upInstance.Spec.OSVersion + kubeVersionSpec = upInstance.Spec.KubeVersion + // osVersion reported by the node from /etc/os-release + osVersion = nodeInstance.Status.NodeInfo.OSImage + ) + upgradeCluster := checkUpgrade(osVersion, osVersionSpec, kubeVersionSpec) if upgradeCluster { if err := r.upgradeNodes(ctx, &upInstance, &nodeInstance); err != nil { return common.RequeueNow, err @@ -91,26 +98,24 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr func (r *UpdateReconciler) upgradeNodes(ctx context.Context, upInstance *housekeeperiov1alpha1.Update, node *corev1.Node) error { - controlPlane := false - if _, ok := node.Labels[constants.LabelMaster]; ok { - controlPlane = true - } if _, ok := node.Labels[constants.LabelUpgrading]; ok { drainer := &drain.Helper{ - Ctx: ctx, - Client: r.KubeClientSet, - GracePeriodSeconds: -1, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.KubeClientSet, + Force: true, + IgnoreAllDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, } if err := drainNode(drainer, node); err != nil { return err } pushInfo := &connection.PushInfo{ - KubeVersion: upInstance.Spec.KubeVersion, - OSImageURL: upInstance.Spec.OSImageURL, - OSVersion: upInstance.Spec.OSVersion, - ControlPlane: controlPlane, + KubeVersion: upInstance.Spec.KubeVersion, + OSImageURL: upInstance.Spec.OSImageURL, + OSVersion: upInstance.Spec.OSVersion, } if err := r.Connection.UpgradeKubeSpec(pushInfo); err != nil { return err @@ -124,13 +129,17 @@ func (r *UpdateReconciler) refreshNodes(ctx context.Context, upInstance *houseke deleteLabel(ctx, r, node) if node.Spec.Unschedulable { drainer := &drain.Helper{ - Ctx: ctx, - Client: r.KubeClientSet, - GracePeriodSeconds: -1, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.KubeClientSet, + Force: true, + IgnoreAllDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, } - if err := drain.cordonOrUncordonNode(false, drainer, node); err != nil { + if err := cordonOrUncordonNode(false, drainer, node); err != nil { + logrus.Errorf("failed to uncordon node %s: %v", node.Name, err) return err } logrus.Infof("uncordon successfully %s node", node.Name) @@ -149,6 +158,7 @@ func deleteLabel(ctx context.Context, r common.ReadWriterClient, node *corev1.No return nil } +// Sets schedulable or not func cordonOrUncordonNode(desired bool, drainer *drain.Helper, node *corev1.Node) error { carry := "cordon" if !desired { @@ -192,15 +202,17 @@ func reqInstance(ctx context.Context, r common.ReadWriterClient, name types.Name return } -func checkUpgrade(node *corev1.Node, osVersionSpec string, kubeVersionSpec string) bool { +// Check if the version is upgraded +func checkUpgrade(osVersion string, osVersionSpec string, kubeVersionSpec string) bool { if len(kubeVersionSpec) > 0 { - labelKubeVersion := fmt.Sprintf("%s%s", constants.LabelKubeVersionPrefix, kubeVersionSpec) - if _, ok := node.Labels[labelKubeVersion]; ok { + markFile := fmt.Sprintf("%s%s%s", "/var/housekeeper/", kubeVersionSpec, ".stamp") + if common.IsFileExist(markFile) { return false } } else { - return node.Status.NodeInfo.OSImage != osVersionSpec + return osVersion != osVersionSpec } + return true } diff --git a/housekeeper/pkg/common/common.go b/housekeeper/pkg/common/common.go index ec07e07..bac1b78 100644 --- a/housekeeper/pkg/common/common.go +++ b/housekeeper/pkg/common/common.go @@ -16,6 +16,7 @@ limitations under the License. package common import ( + "os" "time" ctrl "sigs.k8s.io/controller-runtime" @@ -36,3 +37,14 @@ var ( RequeueNow = ctrl.Result{Requeue: true} RequeueAfter = ctrl.Result{Requeue: true, RequeueAfter: time.Second * 20} ) + +func IsFileExist(path string) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + if fileInfo.IsDir() { + return false + } + return true +} diff --git a/housekeeper/pkg/connection/connection.go b/housekeeper/pkg/connection/connection.go index 3e5c1c6..546e977 100644 --- a/housekeeper/pkg/connection/connection.go +++ b/housekeeper/pkg/connection/connection.go @@ -32,10 +32,9 @@ type Client struct { } type PushInfo struct { - OSImageURL string - OSVersion string - KubeVersion string - ControlPlane bool + OSImageURL string + OSVersion string + KubeVersion string } // Create a grpc channel @@ -58,10 +57,9 @@ func New(socketAddr string) (*Client, error) { func (c *Client) UpgradeKubeSpec(pushInfo *PushInfo) error { _, err := c.client.Upgrade(context.Background(), &pb.UpgradeRequest{ - KubeVersion: pushInfo.KubeVersion, - OsImageUrl: pushInfo.OSImageURL, - OsVersion: pushInfo.OSVersion, - ControlPlane: pushInfo.ControlPlane, + KubeVersion: pushInfo.KubeVersion, + OsImageUrl: pushInfo.OSImageURL, + OsVersion: pushInfo.OSVersion, }) return err } diff --git a/housekeeper/pkg/connection/proto/daemon.pb.go b/housekeeper/pkg/connection/proto/daemon.pb.go index 9ef7338..0446d5b 100644 --- a/housekeeper/pkg/connection/proto/daemon.pb.go +++ b/housekeeper/pkg/connection/proto/daemon.pb.go @@ -44,10 +44,9 @@ type UpgradeRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - KubeVersion string `protobuf:"bytes,1,opt,name=kube_version,json=kubeVersion,proto3" json:"kube_version,omitempty"` - OsImageUrl string `protobuf:"bytes,2,opt,name=os_image_url,json=osImageUrl,proto3" json:"os_image_url,omitempty"` - OsVersion string `protobuf:"bytes,3,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"` - ControlPlane bool `protobuf:"varint,4,opt,name=control_plane,json=controlPlane,proto3" json:"control_plane,omitempty"` + KubeVersion string `protobuf:"bytes,1,opt,name=kube_version,json=kubeVersion,proto3" json:"kube_version,omitempty"` + OsImageUrl string `protobuf:"bytes,2,opt,name=os_image_url,json=osImageUrl,proto3" json:"os_image_url,omitempty"` + OsVersion string `protobuf:"bytes,3,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"` } func (x *UpgradeRequest) Reset() { @@ -103,13 +102,6 @@ func (x *UpgradeRequest) GetOsVersion() string { return "" } -func (x *UpgradeRequest) GetControlPlane() bool { - if x != nil { - return x.ControlPlane - } - return false -} - type UpgradeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -161,27 +153,24 @@ var File_daemon_proto protoreflect.FileDescriptor var file_daemon_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x22, 0x99, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, - 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6b, 0x75, 0x62, - 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x6b, 0x75, 0x62, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, - 0x6f, 0x73, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x73, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1d, - 0x0a, 0x0a, 0x6f, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x6f, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, - 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, - 0x6e, 0x65, 0x22, 0x23, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x32, 0x4e, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, - 0x64, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x07, 0x55, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, - 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x25, 0x5a, 0x23, 0x68, 0x6f, 0x75, 0x73, 0x65, - 0x6b, 0x65, 0x65, 0x70, 0x65, 0x72, 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6b, 0x75, 0x62, 0x65, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6b, 0x75, 0x62, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, 0x6f, + 0x73, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x6f, 0x73, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1d, 0x0a, + 0x0a, 0x6f, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x6f, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x23, 0x0a, 0x0f, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, + 0x72, 0x32, 0x4e, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x16, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x25, 0x5a, 0x23, 0x68, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x65, 0x72, + 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/housekeeper/pkg/connection/proto/daemon.proto b/housekeeper/pkg/connection/proto/daemon.proto index 649f714..f4f5c44 100644 --- a/housekeeper/pkg/connection/proto/daemon.proto +++ b/housekeeper/pkg/connection/proto/daemon.proto @@ -28,7 +28,6 @@ message UpgradeRequest { string kube_version = 1; string os_image_url = 2; string os_version = 3; - bool control_plane = 4; } message UpgradeResponse { diff --git a/housekeeper/pkg/constants/constants.go b/housekeeper/pkg/constants/constants.go index ea930a4..4528bda 100644 --- a/housekeeper/pkg/constants/constants.go +++ b/housekeeper/pkg/constants/constants.go @@ -18,8 +18,6 @@ package constants const ( // LabelUpgrading is the key of the upgrading label for nodes LabelUpgrading = "upgrade.housekeeper.io/upgrading" - // LabelKubeVersionPrefix defines the label associated with kubernetes version - LabelKubeVersionPrefix = "upgrade.kubernetes.version.io/" // LabelMaster defines the label associated with master node. LabelMaster = "node-role.kubernetes.io/master" ) @@ -29,4 +27,3 @@ const ( SockDir = "/run/housekeeper-daemon" SockName = "housekeeper-daemon.sock" ) - -- Gitee From 548483b773bfaa873611d2898afd832feef4ff60 Mon Sep 17 00:00:00 2001 From: duyiwei Date: Wed, 12 Jul 2023 16:09:17 +0800 Subject: [PATCH 25/38] add logrus section ;add CertDirectory in CertificateManager --- pkg/cert/cacert.go | 8 +++++++- pkg/cert/certapi.go | 1 + pkg/cert/certs.go | 7 +++++++ pkg/cert/tools.go | 28 ++++++++++++++++++++++------ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/pkg/cert/cacert.go b/pkg/cert/cacert.go index 55a5c53..1607ac6 100644 --- a/pkg/cert/cacert.go +++ b/pkg/cert/cacert.go @@ -23,6 +23,8 @@ import ( "crypto/x509/pkix" "math/big" "time" + + "github.com/sirupsen/logrus" ) func (cm *CertificateManager) GenerateCACertificate() error { @@ -31,6 +33,7 @@ func (cm *CertificateManager) GenerateCACertificate() error { var err error cm.CAKey, err = rsa.GenerateKey(rand.Reader, 2048) if err != nil { + logrus.Errorf("Failed to generate CA privatekey: %v", err) return err } caPublicKey := &cm.CACert.PublicKey @@ -39,6 +42,7 @@ func (cm *CertificateManager) GenerateCACertificate() error { serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { + logrus.Errorf("Failed to generate serialNumber: %v", err) return err } now := time.Now() @@ -61,14 +65,16 @@ func (cm *CertificateManager) GenerateCACertificate() error { //caCertBytes是生成证书的中间步骤，它用于将证书的二进制表示存储在内存中，以便后续操作可以使用它 caCertBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caPublicKey, cm.CAKey) if err != nil { + logrus.Errorf("Failed to create CA Certificate(caCertBytes): %v", err) return err } //cm.CACert表示已经生成的CA证书，用于存储CA证书的详细信息，例如证书序列号、主题、有效期等 cm.CACert, err = x509.ParseCertificate(caCertBytes) if err != nil { + logrus.Errorf("Failed to parse CA Certificate: %v", err) return err } - + logrus.Infof("Successfully generate CA certificate") return nil } diff --git a/pkg/cert/certapi.go b/pkg/cert/certapi.go index a28444e..4170ed6 100644 --- a/pkg/cert/certapi.go +++ b/pkg/cert/certapi.go @@ -33,4 +33,5 @@ type CertificateManager struct { ComponentKey *rsa.PrivateKey ComponentCert *x509.Certificate ValidDays int + CertDirectory string } diff --git a/pkg/cert/certs.go b/pkg/cert/certs.go index 838d74d..fe1b4b4 100644 --- a/pkg/cert/certs.go +++ b/pkg/cert/certs.go @@ -23,6 +23,8 @@ import ( "crypto/x509/pkix" "math/big" "time" + + "github.com/sirupsen/logrus" ) // 使用CA证书和私钥生成组件证书和私钥 @@ -32,6 +34,7 @@ func (cm *CertificateManager) GenerateComponentCertificate(componentName string) var err error cm.ComponentKey, err = rsa.GenerateKey(rand.Reader, 2048) if err != nil { + logrus.Errorf("Failed to generate %s privatekey: %v", componentName, err) return err } componentPublicKey := &cm.ComponentKey.PublicKey @@ -40,6 +43,7 @@ func (cm *CertificateManager) GenerateComponentCertificate(componentName string) serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { + logrus.Errorf("Failed to generate %s serialNumber: %v", componentName, err) return err } now := time.Now() @@ -57,13 +61,16 @@ func (cm *CertificateManager) GenerateComponentCertificate(componentName string) // 使用CA证书和私钥生成组件证书 componentCertBytes, err := x509.CreateCertificate(rand.Reader, componentTemplate, cm.CACert, componentPublicKey, cm.CAKey) if err != nil { + logrus.Errorf("Failed to create %s Certificate(caCertBytes): %v", componentName, err) return err } cm.ComponentCert, err = x509.ParseCertificate(componentCertBytes) if err != nil { + logrus.Errorf("Failed to parse %s Certificate: %v", componentName, err) return err } + logrus.Infof("Successfully generate %s certificate", componentName) return nil } diff --git a/pkg/cert/tools.go b/pkg/cert/tools.go index 9eb5a26..e63826c 100644 --- a/pkg/cert/tools.go +++ b/pkg/cert/tools.go @@ -20,9 +20,9 @@ import ( "crypto/x509" "encoding/pem" "io/ioutil" -) -const certpath = "/etc/kubernetes/pki/" + "github.com/sirupsen/logrus" +) // CACertPEM 返回CA证书的PEM格式字节切片 func (cm *CertificateManager) CACertPEM() []byte { @@ -57,11 +57,27 @@ func (cm *CertificateManager) ComponentKeyPEM() []byte { } // SaveCertificateToFile 将证书保存到文件 -func SaveCertificateToFile(filename string, certPEM []byte) error { - return ioutil.WriteFile(certpath+filename, certPEM, 0644) +func (cm *CertificateManager) SaveCertificateToFile(filename string, certPEM []byte) error { + err := ioutil.WriteFile(cm.CertDirectory+"/"+filename, certPEM, 0644) + if err != nil { + logrus.Errorf("Faile to save %s: %v", filename, err) + return err + } + + logrus.Infof("Successfully saved %s", filename) + + return nil } // SavePrivateKeyToFile 将私钥保存到文件 -func SavePrivateKeyToFile(filename string, keyPEM []byte) error { - return ioutil.WriteFile(certpath+filename, keyPEM, 0600) +func (cm *CertificateManager) SavePrivateKeyToFile(filename string, keyPEM []byte) error { + err := ioutil.WriteFile(cm.CertDirectory+"/"+filename, keyPEM, 0600) + if err != nil { + logrus.Errorf("Faile to save %s: %v", filename, err) + return err + } + + logrus.Infof("Successfully saved %s", filename) + + return nil } -- Gitee From 1e380ce251c2e433ee4e8bbe8d5edfc0a3d301af Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Thu, 13 Jul 2023 15:15:03 +0800 Subject: [PATCH 26/38] modify terraform functions --- pkg/infra/terraform/init.go | 29 +++++-------------- pkg/infra/terraform/terraform.go | 49 ++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 43 deletions(-) diff --git a/pkg/infra/terraform/init.go b/pkg/infra/terraform/init.go index e52a13b..cbd3ad3 100644 --- a/pkg/infra/terraform/init.go +++ b/pkg/infra/terraform/init.go @@ -20,37 +20,22 @@ import ( "context" "path/filepath" - prov "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/providers" "github.com/hashicorp/terraform-exec/tfexec" - "github.com/openshift/installer/data" "github.com/pkg/errors" ) -func unpack(workingDir string, platform string, target string) (err error) { - err = data.Unpack(workingDir, filepath.Join(platform, target)) - if err != nil { - return err - } - - return nil -} - // terraform init -func TFInit(workingDir string, platform string, target string, terraformBinary string, providers []prov.Provider) (err error) { - err = unpack(workingDir, platform, target) +func TFInit(dir string, terraformDir string) error { + tf, err := newTFExec(dir, terraformDir) if err != nil { - return errors.Wrap(err, "failed to unpack terraform modules") + return errors.Wrap(err, "failed to create a new tfexec") } - tf, err := newTFExec(workingDir, terraformBinary) + // 使用本地terraform插件 + err = tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformDir, "plugins"))) if err != nil { - return errors.Wrap(err, "failed to create a new tfexec.") + return errors.Wrap(err, "failed to init terraform") } - // 如果想导入本地已有插件，需构建相应目录 - // 如openstack插件所需目录为plugins/registry.terraform.io/terraform-provider-openstack/openstack/1.51.1/linux_arm64/terraform-provider-openstack_v1.51.1 - return errors.Wrap( - tf.Init(context.Background(), tfexec.PluginDir(filepath.Join(terraformBinary, "plugins"))), - "failed doing terraform init.", - ) + return nil } diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index ba540da..f114dda 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -20,6 +20,7 @@ import ( "context" "os" "path/filepath" + "runtime" "github.com/hashicorp/terraform-exec/tfexec" "github.com/openshift/installer/pkg/lineprinter" @@ -27,21 +28,21 @@ import ( "github.com/sirupsen/logrus" ) -func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, error) { - execPath := filepath.Join(terraformBinary, "bin", "terraform") - tf, err := tfexec.NewTerraform(workingDir, execPath) +func newTFExec(dir string, terraformDir string) (*tfexec.Terraform, error) { + tfPath := filepath.Join(terraformDir, "bin", runtime.GOOS+"_"+runtime.GOARCH, "terraform") + tf, err := tfexec.NewTerraform(dir, tfPath) if err != nil { return nil, err } // If the log path is not set, terraform will not receive debug logs. - if logPath, ok := os.LookupEnv("TEREFORM_LOG_PATH"); ok { - if err := tf.SetLog(os.Getenv("TEREFORM_LOG")); err != nil { + if logPath, ok := os.LookupEnv("TERRAFORM_LOG_PATH"); ok { + if err := tf.SetLog(os.Getenv("TERRAFORM_LOG")); err != nil { logrus.Infof("Skipping setting terraform log levels: %v", err) } else { - tf.SetLogCore(os.Getenv("TEREFORM_LOG_CORE")) //nolint:errcheck - tf.SetLogProvider(os.Getenv("TEREFORM_LOG_PROVIDER")) //nolint:errcheck - tf.SetLogPath(logPath) //nolint:errcheck + tf.SetLogCore(os.Getenv("TERRAFORM_LOG_CORE")) //nolint:errcheck + tf.SetLogProvider(os.Getenv("TERRAFORM_LOG_PROVIDER")) //nolint:errcheck + tf.SetLogPath(logPath) //nolint:errcheck } } @@ -59,33 +60,39 @@ func newTFExec(workingDir string, terraformBinary string) (*tfexec.Terraform, er } // terraform apply -func TFApply(workingDir string, platform string, stage Stage, terraformBinary string, applyOpts ...tfexec.ApplyOption) error { - if err := TFInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { +func TFApply(dir string, terraformDir string, applyOpts ...tfexec.ApplyOption) error { + if err := TFInit(dir, terraformDir); err != nil { return err } - tf, err := newTFExec(workingDir, terraformBinary) + tf, err := newTFExec(dir, terraformDir) if err != nil { - return errors.Wrap(err, "failed to create a new tfexec.") + return errors.Wrap(err, "failed to create a new tfexec") } err = tf.Apply(context.Background(), applyOpts...) - return errors.Wrap(err, "failed to apply Terraform.") + if err != nil { + return errors.Wrap(err, "failed to apply Terraform") + } + + return nil } // terraform destroy -func TFDestroy(workingDir string, platform string, stage Stage, terraformBinary string, destroyOpts ...tfexec.DestroyOption) error { - if err := TFInit(workingDir, platform, stage.Name(), terraformBinary, stage.Providers()); err != nil { +func TFDestroy(dir string, terraformDir string, destroyOpts ...tfexec.DestroyOption) error { + if err := TFInit(dir, terraformDir); err != nil { return err } - tf, err := newTFExec(workingDir, terraformBinary) + tf, err := newTFExec(dir, terraformDir) + if err != nil { + return errors.Wrap(err, "failed to destroy a new tfexec") + } + + err = tf.Destroy(context.Background(), destroyOpts...) if err != nil { - return errors.Wrap(err, "failed to create a new tfexec.") + return errors.Wrap(err, "failed to destroy terraform") } - return errors.Wrap( - tf.Destroy(context.Background(), destroyOpts...), - "failed doing terraform destroy.", - ) + return nil } -- Gitee From dd13e9e7fe77b03a125b855042f544a4e74e25e3 Mon Sep 17 00:00:00 2001 From: lauk Date: Fri, 14 Jul 2023 15:18:58 +0800 Subject: [PATCH 27/38] add the evictPodForce configuration option --- .../operator/api/v1alpha1/update_types.go | 7 ++++--- .../config/crd/housekeeper.io_updates.yaml | 4 ++++ .../controllers/update_controller.go | 17 ++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/housekeeper/operator/api/v1alpha1/update_types.go b/housekeeper/operator/api/v1alpha1/update_types.go index 7351fd4..300e950 100644 --- a/housekeeper/operator/api/v1alpha1/update_types.go +++ b/housekeeper/operator/api/v1alpha1/update_types.go @@ -27,9 +27,10 @@ import ( type UpdateSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - OSVersion string `json:"osVersion"` - OSImageURL string `json:"osImageURL"` - KubeVersion string `json:"kubeVersion"` + OSVersion string `json:"osVersion"` + OSImageURL string `json:"osImageURL"` + KubeVersion string `json:"kubeVersion"` + EvictPodForce bool `json:"evictPodForce"` } // UpdateStatus defines the observed state of Update diff --git a/housekeeper/operator/config/crd/housekeeper.io_updates.yaml b/housekeeper/operator/config/crd/housekeeper.io_updates.yaml index 53cf004..2938aa0 100644 --- a/housekeeper/operator/config/crd/housekeeper.io_updates.yaml +++ b/housekeeper/operator/config/crd/housekeeper.io_updates.yaml @@ -44,10 +44,14 @@ spec: osVersion: description: 'The version used to upgrade OS' type: string + evictPodForce: + description: 'If true, force evict the pod' + type: boolean required: - kubeVersion - osImageURL - osVersion + - evictPodForce type: object status: description: UpdateStatus defines the observed state of Update diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go index 6546451..ea4e428 100644 --- a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -102,13 +102,15 @@ func (r *UpdateReconciler) upgradeNodes(ctx context.Context, upInstance *houseke drainer := &drain.Helper{ Ctx: ctx, Client: r.KubeClientSet, - Force: true, IgnoreAllDaemonSets: true, DeleteEmptyDirData: true, GracePeriodSeconds: -1, Out: os.Stdout, ErrOut: os.Stderr, } + if upInstance.Spec.EvictPodForce { + drainer.Force = true + } if err := drainNode(drainer, node); err != nil { return err } @@ -129,14 +131,11 @@ func (r *UpdateReconciler) refreshNodes(ctx context.Context, upInstance *houseke deleteLabel(ctx, r, node) if node.Spec.Unschedulable { drainer := &drain.Helper{ - Ctx: ctx, - Client: r.KubeClientSet, - Force: true, - IgnoreAllDaemonSets: true, - DeleteEmptyDirData: true, - GracePeriodSeconds: -1, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.KubeClientSet, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, } if err := cordonOrUncordonNode(false, drainer, node); err != nil { logrus.Errorf("failed to uncordon node %s: %v", node.Name, err) -- Gitee From 768ff1f14e2948dc82b12768c6e69994c7980b3f Mon Sep 17 00:00:00 2001 From: lauk Date: Fri, 14 Jul 2023 16:55:21 +0800 Subject: [PATCH 28/38] daemon code functionality optimization --- housekeeper/daemon/server/server.go | 85 ++++++++++++----------------- 1 file changed, 36 insertions(+), 49 deletions(-) diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go index a85f234..faa24e2 100644 --- a/housekeeper/daemon/server/server.go +++ b/housekeeper/daemon/server/server.go @@ -31,11 +31,12 @@ import ( ) const ( - ostreeImage = "ostree-unverified-image:docker://" - kubeadmCmd = "/usr/local/bin/kubeadm" - upgradeControlPlaneCmd = "/usr/local/bin/kubeadm upgrade apply -y" - upgradeNodesCmd = "/usr/local/bin/kubeadm upgrade node" - kubeletUpdateCmd = "systemctl daemon-reload && systemctl restart kubelet" + ostreeImage = "ostree-unverified-image:docker://" + kubeadmCmd = "/usr/local/bin/kubeadm" + upgradeMasterCmd = "/usr/local/bin/kubeadm upgrade apply -y" + upgradeWorkerCmd = "/usr/local/bin/kubeadm upgrade node" + kubeletUpdateCmd = "systemctl daemon-reload && systemctl restart kubelet" + adminFile = "/etc/kubernetes/admin.conf" ) type Server struct { @@ -47,7 +48,7 @@ type Server struct { func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.UpgradeResponse, error) { s.mu.Lock() defer s.mu.Unlock() - markFile := fmt.Sprintf("%s%s%s", "/var/housekeeper/", req.KubeVersion, ".stamp") + // upgrade os if len(req.OsVersion) > 0 { //Checking for os version @@ -55,20 +56,21 @@ func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.Upgrade return &pb.UpgradeResponse{}, err } } - if common.IsFileExist(markFile) { - return &pb.UpgradeResponse{}, nil - } // upgrade kubernetes if len(req.KubeVersion) > 0 { + markFile := fmt.Sprintf("%s%s%s", "/var/housekeeper/", req.KubeVersion, ".stamp") + if common.IsFileExist(markFile) { + return &pb.UpgradeResponse{}, nil + } //Checking for kubernetes version if err := checkKubeVersion(req); err != nil { return &pb.UpgradeResponse{}, err } - } - // todo: check upgrade successfully - if err := markNode(markFile); err != nil { - logrus.Errorf("failed to mark node: %v", err) - return &pb.UpgradeResponse{}, err + // todo: check upgrade successfully + if err := markNode(markFile); err != nil { + logrus.Errorf("failed to mark node: %v", err) + return &pb.UpgradeResponse{}, err + } } return &pb.UpgradeResponse{}, nil } @@ -138,17 +140,13 @@ func upgradeOSVersion(req *pb.UpgradeRequest) error { } func upgradeKubeVersion(req *pb.UpgradeRequest) error { - // todo: pull images before performing the upgrade - kubeVersion := req.KubeVersion - if ok, err := isControlPlaneNode(); err != nil { - return err - } else if ok { - if err = upgradeControlPlaneNode(kubeVersion); err != nil { + if isMasterNode() { + if err := upgradeMasterNodes(req.KubeVersion); err != nil { logrus.Errorf("failed to upgrade master nodes: %v", err) return err } } else { - if err = upgradeNodes(); err != nil { + if err := upgradeWorkerNodes(); err != nil { logrus.Errorf("failed to upgrade worker nodes: %v", err) return err } @@ -156,51 +154,40 @@ func upgradeKubeVersion(req *pb.UpgradeRequest) error { return nil } -func upgradeControlPlaneNode(version string) error { - args := []string{"-c", upgradeControlPlaneCmd, version} - if err := exec.Command("/bin/sh", args...).Run(); err != nil { - logrus.Errorf("failed to upgrade nodes: %w", err) - return err - } +func upgradeMasterNodes(version string) error { if err := exec.Command("/bin/sh", "-c", kubeletUpdateCmd).Run(); err != nil { logrus.Errorf("failed to restart kubelet: %w", err) return err } - return nil -} - -func upgradeNodes() error { - if err := exec.Command("/bin/sh", "-c", upgradeNodesCmd).Run(); err != nil { + args := []string{"-c", upgradeMasterCmd, version} + if err := exec.Command("/bin/sh", args...).Run(); err != nil { logrus.Errorf("failed to upgrade nodes: %w", err) return err } + return nil +} + +func upgradeWorkerNodes() error { if err := exec.Command("/bin/sh", "-c", kubeletUpdateCmd).Run(); err != nil { logrus.Errorf("failed to restart kubelet: %w", err) return err } + if err := exec.Command("/bin/sh", "-c", upgradeWorkerCmd).Run(); err != nil { + logrus.Errorf("failed to upgrade nodes: %w", err) + return err + } return nil } -func isControlPlaneNode() (bool, error) { - if !common.IsFileExist("/etc/kubernetes/admin.conf") { - return false, nil - } - ipArgs := []string{"-c", "ifconfig | grep 'inet' | grep 'broadcast'| awk '{print $2}'"} - ipAddress, err := runCmd("/bin/sh", ipArgs...) - if err != nil { - return false, err - } - adminArgs := []string{"-c", "cat /etc/kubernetes/admin.conf | grep 'server' |grep -E -o '([0-9]{1,3}.){3}[0-9]{1,3}'"} - ipControlPlane, err := runCmd("/bin/sh", adminArgs...) - if err != nil { - return false, err - } - return string(ipControlPlane) == string(ipAddress), nil +func isMasterNode() bool { + return common.IsFileExist(adminFile) } func markNode(file string) error { - if err := os.MkdirAll("/var/housekeeper", 0644); err != nil { - return err + if !common.IsFileExist("/var/housekeeper") { + if err := os.MkdirAll("/var/housekeeper", 0644); err != nil { + return err + } } args := []string{"-c", "touch", file} _, err := runCmd("/bin/sh", args...) -- Gitee From ad2ec2ce1f808c68779b55dee9c9672aa5077d04 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Fri, 14 Jul 2023 17:28:46 +0800 Subject: [PATCH 29/38] modify instance-create flow --- pkg/infra/assets/cluster/cluster.go | 89 ++++++++----------- pkg/infra/terraform/stage.go | 6 -- .../terraform/stages/openstack/stages.go | 4 +- pkg/infra/terraform/terraform.go | 6 +- 4 files changed, 44 insertions(+), 61 deletions(-) diff --git a/pkg/infra/assets/cluster/cluster.go b/pkg/infra/assets/cluster/cluster.go index c22963e..fdea808 100644 --- a/pkg/infra/assets/cluster/cluster.go +++ b/pkg/infra/assets/cluster/cluster.go @@ -22,97 +22,86 @@ import ( "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/assets" "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform" - platformStages "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/stages/platform" "github.com/hashicorp/terraform-exec/tfexec" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +/* + installDir：自定义工作目录，在该路径下自动创建terraform文件夹 + dir：该路径存放tf配置文件 + terraformDir：该路径包含bin、plugins目录，存放terraform执行文件以及所需plugins +*/ + +type InfraProvider interface { + Create() +} + type Cluster struct { - FileList []*assets.File + InstallDir string + Platform string + Name string } -func (c *Cluster) Create(platform string, node string, installDir string) (err error) { - if installDir == "" { - logrus.Fatal("InstallDir has not been set") - } +// TODO: 配置文件准备阶段 - terraformVariables := &TerraformVariables{} - stages, err := platformStages.StagesForPlatform(platform, node) +func (c *Cluster) Create() error { + terraformDir := filepath.Join(c.InstallDir, "terraform") + dir := filepath.Join(terraformDir, c.Platform, c.Name) - terraformBinary := filepath.Join(installDir, "terraform") - _, err = os.Stat(terraformBinary) - if os.IsNotExist(err) { - if err := os.Mkdir(terraformBinary, 0777); err != nil { - return errors.Wrap(err, "could not create the terraform directory") - } - } + terraformVariables := &TerraformVariables{} + tfvarsFiles := make([]*assets.File, 0, len(terraformVariables.Files())+len(c.Platform)+len(c.Name)) + tfvarsFiles = append(tfvarsFiles, terraformVariables.Files()...) - // TODO 将所需二进制文件保存到terraformBinary目录 + logrus.Infof("start to create %s in %s", c.Name, c.Platform) - terraformBinaryPath, err := filepath.Abs(terraformBinary) + outputs, err := c.applyStage(dir, terraformDir, tfvarsFiles) if err != nil { - return errors.Wrap(err, "cannot get absolute path of terraform directory") + return errors.Wrapf(err, "failed to create %s in %s", c.Name, c.Platform) } - tfvarsFiles := make([]*assets.File, 0, len(terraformVariables.Files())+len(stages)) - tfvarsFiles = append(tfvarsFiles, terraformVariables.Files()...) - - for _, stage := range stages { - outputs, err := c.applyStage(platform, stage, terraformBinaryPath, tfvarsFiles) - if err != nil { - return errors.Wrapf(err, "failure applying terraform for %q stage", stage.Name()) - } - tfvarsFiles = append(tfvarsFiles, outputs) - c.FileList = append(c.FileList, outputs) - } + logrus.Info(string(outputs.Data)) + logrus.Infof("succeed in creating %s in %s", c.Name, c.Platform) return nil } -func (c *Cluster) applyStage(platform string, stage terraform.Stage, terraformBinary string, tfvarsFiles []*assets.File) (*assets.File, error) { - tmpDir := filepath.Join(terraformBinary, platform, stage.Name()) - if err := os.MkdirAll(tmpDir, 0777); err != nil { - return nil, errors.Wrapf(err, "cannot create the directory for %s", stage.Name()) - } - +func (c *Cluster) applyStage(dir string, terraformDir string, tfvarsFiles []*assets.File) (*assets.File, error) { var applyOpts []tfexec.ApplyOption for _, file := range tfvarsFiles { - if err := os.WriteFile(filepath.Join(tmpDir, file.Filename), file.Data, 0o600); err != nil { + if err := os.WriteFile(filepath.Join(dir, file.Filename), file.Data, 0o600); err != nil { return nil, err } - applyOpts = append(applyOpts, tfexec.VarFile(filepath.Join(tmpDir, file.Filename))) + applyOpts = append(applyOpts, tfexec.VarFile(filepath.Join(dir, file.Filename))) } - return c.applyTerraform(tmpDir, platform, stage, terraformBinary, applyOpts...) + return c.applyTerraform(dir, terraformDir, applyOpts...) } -func (c *Cluster) applyTerraform(tmpDir string, platform string, stage terraform.Stage, terraformBinary string, applyOpts ...tfexec.ApplyOption) (*assets.File, error) { - applyErr := terraform.TFApply(tmpDir, platform, stage, terraformBinary, applyOpts...) +func (c *Cluster) applyTerraform(dir string, terraformDir string, applyOpts ...tfexec.ApplyOption) (*assets.File, error) { + applyErr := terraform.TFApply(dir, terraformDir, applyOpts...) - if data, err := os.ReadFile(filepath.Join(tmpDir, terraform.StateFilename)); err == nil { - c.FileList = append(c.FileList, &assets.File{ - Filename: stage.StateFilename(), - Data: data, - }) - } else if !os.IsNotExist(err) { + _, err := os.Stat(filepath.Join(dir, "terraform.tfstate")) + if os.IsNotExist(err) { logrus.Errorf("Failed to read tfstate: %v", err) return nil, errors.Wrap(err, "failed to read tfstate") } if applyErr != nil { - return nil, errors.WithMessage(applyErr, "failed to apply Terraform.") + return nil, errors.WithMessage(applyErr, "failed to apply Terraform") } - outputs, err := terraform.Outputs(tmpDir, terraformBinary) + outputs, err := terraform.Outputs(dir, terraformDir) if err != nil { - return nil, errors.Wrapf(err, "could not get outputs from stage %q", stage.Name()) + return nil, errors.Wrap(err, "could not get outputs file") } outputsFile := &assets.File{ - Filename: stage.OutputsFilename(), + Filename: "outputs", Data: outputs, } return outputsFile, nil } + +// TODO：(c *Cluster) Destroy diff --git a/pkg/infra/terraform/stage.go b/pkg/infra/terraform/stage.go index 1fbe53d..dabe48f 100644 --- a/pkg/infra/terraform/stage.go +++ b/pkg/infra/terraform/stage.go @@ -21,12 +21,6 @@ import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/terraform/provi type Stage interface { Name() string - // terraform state file - StateFilename() string - - // outputs file - OutputsFilename() string - // the list of providers that are used for the stage Providers() []providers.Provider } diff --git a/pkg/infra/terraform/stages/openstack/stages.go b/pkg/infra/terraform/stages/openstack/stages.go index 93ad003..1b1f18a 100644 --- a/pkg/infra/terraform/stages/openstack/stages.go +++ b/pkg/infra/terraform/stages/openstack/stages.go @@ -8,10 +8,10 @@ import ( var PlatformStages = []terraform.Stage{} -func AddPlatformStage(stage string) { +func AddPlatformStage(name string) { newStage := stages.NewStage( "openstack", - stage, + name, []providers.Provider{providers.OpenStack}, ) diff --git a/pkg/infra/terraform/terraform.go b/pkg/infra/terraform/terraform.go index f114dda..6bb6a9c 100644 --- a/pkg/infra/terraform/terraform.go +++ b/pkg/infra/terraform/terraform.go @@ -47,12 +47,12 @@ func newTFExec(dir string, terraformDir string) (*tfexec.Terraform, error) { } // Add terraform info logs to the installer log - lpPrint := &lineprinter.LinePrinter{Print: (&lineprinter.Trimmer{WrappedPrint: logrus.Print}).Print} + lpDebug := &lineprinter.LinePrinter{Print: (&lineprinter.Trimmer{WrappedPrint: logrus.Debug}).Print} lpError := &lineprinter.LinePrinter{Print: (&lineprinter.Trimmer{WrappedPrint: logrus.Error}).Print} - defer lpPrint.Close() + defer lpDebug.Close() defer lpError.Close() - tf.SetStdout(lpPrint) + tf.SetStdout(lpDebug) tf.SetStderr(lpError) tf.SetLogger(newPrintfer()) -- Gitee From 14fc882cbb1272522a410fddd83b72489a9b520f Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Mon, 17 Jul 2023 16:50:43 +0800 Subject: [PATCH 30/38] add cluster destroy function in infra --- pkg/infra/assets/cluster/cluster.go | 45 ++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/pkg/infra/assets/cluster/cluster.go b/pkg/infra/assets/cluster/cluster.go index fdea808..2fe19e5 100644 --- a/pkg/infra/assets/cluster/cluster.go +++ b/pkg/infra/assets/cluster/cluster.go @@ -35,6 +35,7 @@ import ( type InfraProvider interface { Create() + Destroy() } type Cluster struct { @@ -104,4 +105,46 @@ func (c *Cluster) applyTerraform(dir string, terraformDir string, applyOpts ...t return outputsFile, nil } -// TODO：(c *Cluster) Destroy +func (c *Cluster) Destroy() error { + terraformDir := filepath.Join(c.InstallDir, "terraform") + dir := filepath.Join(terraformDir, c.Platform, c.Name) + + logrus.Infof("start to destroy %s in %s", c.Name, c.Platform) + + // TODO: Destroy的tfvarsFiles的获取 + + // terraformVariables := &TerraformVariables{} + // tfvarsFiles := make([]*assets.File, 0, len(terraformVariables.Files())+len(c.Platform)+len(c.Name)) + // tfvarsFiles = append(tfvarsFiles, terraformVariables.Files()...) + + err := c.destroyStage(dir, terraformDir, tfvarsFiles) + if err != nil { + return errors.Wrapf(err, "failed to destroy %s in %s", c.Name, c.Platform) + } + os.Remove(dir) + + logrus.Infof("succeed in destroying %s in %s", c.Name, c.Platform) + + return nil +} + +func (c *Cluster) destroyStage(dir string, terraformDir string, tfvarsFiles []*assets.File) error { + var destroyOpts []tfexec.DestroyOption + for _, file := range tfvarsFiles { + if err := os.WriteFile(filepath.Join(dir, file.Filename), file.Data, 0o600); err != nil { + return err + } + destroyOpts = append(destroyOpts, tfexec.VarFile(filepath.Join(dir, file.Filename))) + } + + return destroyTerraform(dir, terraformDir, destroyOpts...) +} + +func destroyTerraform(dir string, terraformDir string, destroyOpts ...tfexec.DestroyOption) error { + destroyErr := terraform.TFDestroy(dir, terraformDir, destroyOpts...) + if destroyErr != nil { + return errors.WithMessage(destroyErr, "failed to destroy Terraform") + } + + return nil +} -- Gitee From b96de25652d05b2469e6e23c80302f3dc8f163a7 Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 19 Jul 2023 15:51:36 +0800 Subject: [PATCH 31/38] add the MaxUnavailable configuration option --- .../controllers/update_controller.go | 169 ++++++++++++------ housekeeper/pkg/constants/constants.go | 4 + 2 files changed, 122 insertions(+), 51 deletions(-) diff --git a/housekeeper/operator/housekeeper-operator/controllers/update_controller.go b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go index d202332..0cc7af8 100644 --- a/housekeeper/operator/housekeeper-operator/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "time" "github.com/sirupsen/logrus" housekeeperiov1alpha1 "housekeeper.io/operator/api/v1alpha1" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -60,59 +62,86 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return common.NoRequeue, nil } ctx = context.Background() - err := setLabels(ctx, r, req) + //返回worker节点的数量 + upInstance, nodesNum, err := getUpgradeInstance(ctx, r, req.NamespacedName) if err != nil { + return common.RequeueNow, err + } + limit := min(upInstance.Spec.MaxUnavailable, nodesNum) + if requeueAfter, err := setLabels(ctx, r, req, limit, upInstance); err != nil { logrus.Errorf("unable set nodes label: %v", err) return common.RequeueNow, err + } else if requeueAfter { + return common.RequeueAfter, nil } - return common.RequeueAfter, nil + return common.RequeueNow, nil } -func setLabels(ctx context.Context, r common.ReadWriterClient, req ctrl.Request) error { +func getUpgradeInstance(ctx context.Context, r common.ReadWriterClient, name types.NamespacedName) ( + upInstance housekeeperiov1alpha1.Update, nodeNum int, err error) { + if err = r.Get(ctx, name, &upInstance); err != nil { + logrus.Errorf("unable to fetch upgrade instance: %v", err) + return + } + requirement, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) + if err != nil { + logrus.Errorf("unable to create requirement %s: %v"+constants.LabelMaster, err) + return + } + nodesItems, err := getNodes(ctx, r, 0, *requirement) + if err != nil { + logrus.Errorf("failed to get nodes list: %v", err) + return + } + nodeNum = len(nodesItems) + return +} + +func setLabels(ctx context.Context, r common.ReadWriterClient, req ctrl.Request, limit int, + upInstance housekeeperiov1alpha1.Update) (bool, error) { reqUpgrade, err := labels.NewRequirement(constants.LabelUpgrading, selection.DoesNotExist, nil) if err != nil { logrus.Errorf("unable to create upgrade label requirement: %v", err) - return err + return false, err } reqMaster, err := labels.NewRequirement(constants.LabelMaster, selection.Exists, nil) if err != nil { logrus.Errorf("unable to create master label requirement: %v", err) - return err + return false, err } reqNoMaster, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) if err != nil { logrus.Errorf("unable to create non-master label requirement: %v", err) - return err + return false, err } - masterNodes, err := getNodes(ctx, r, *reqUpgrade, *reqMaster) + masterNodes, err := getNodes(ctx, r, 1, *reqUpgrade, *reqMaster) if err != nil { logrus.Errorf("unable to get master node list: %v", err) - return err + return false, err } - noMasterNodes, err := getNodes(ctx, r, *reqUpgrade, *reqNoMaster) + //limit: 限制worker节点每次升级的数量 + noMasterNodes, err := getNodes(ctx, r, limit, *reqUpgrade, *reqNoMaster) if err != nil { logrus.Errorf("unable to get non-master node list: %v", err) - return err + return false, err } - upgradeCompleted, err := assignUpdated(ctx, r, masterNodes, req.NamespacedName) + needRequeue, err := assignUpdated(ctx, r, masterNodes, upInstance) if err != nil { - logrus.Errorf("unabel to add the label of the master nodes: %v", err) - return err - } - //Make sure the master upgrade is complete before start upgrading non-master nodes - if upgradeCompleted { - _, err := assignUpdated(ctx, r, noMasterNodes, req.NamespacedName) - if err != nil { - logrus.Errorf("unabel to add the label of non-master nodes: %v", err) - return err - } + logrus.Errorf("unabel to add upgrade label of the master nodes: %v", err) + return false, err + } else if needRequeue { + return true, nil } - return nil + if needRequeue, err = assignUpdated(ctx, r, noMasterNodes, upInstance); err != nil { + logrus.Errorf("unabel to add upgrade label of non-master nodes: %v", err) + return false, err + } + return needRequeue, nil } -func getNodes(ctx context.Context, r common.ReadWriterClient, reqs ...labels.Requirement) ([]corev1.Node, error) { +func getNodes(ctx context.Context, r common.ReadWriterClient, limit int, reqs ...labels.Requirement) ([]corev1.Node, error) { var nodeList corev1.NodeList - opts := client.ListOptions{LabelSelector: labels.NewSelector().Add(reqs...)} + opts := client.ListOptions{LabelSelector: labels.NewSelector().Add(reqs...), Limit: int64(limit)} if err := r.List(ctx, &nodeList, &opts); err != nil { logrus.Errorf("unable to list nodes with requirements: %v", err) return nil, err @@ -121,14 +150,9 @@ func getNodes(ctx context.Context, r common.ReadWriterClient, reqs ...labels.Req } // Add the label to nodes -func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []corev1.Node, name types.NamespacedName) (bool, error) { - var upInstance housekeeperiov1alpha1.Update - if err := r.Get(ctx, name, &upInstance); err != nil { - logrus.Errorf("unable to get update Instance %v", err) - return false, err - } +func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []corev1.Node, + upInstance housekeeperiov1alpha1.Update) (bool, error) { var ( - upgradeNum = -1 kubeVersionSpec = upInstance.Spec.KubeVersion osVersionSpec = upInstance.Spec.OSVersion ) @@ -136,33 +160,76 @@ func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []co logrus.Warning("os version is required") return false, nil } + if len(nodeList) == 0 { + return false, nil + } for _, node := range nodeList { - var ( - kubeProxyVersion = node.Status.NodeInfo.KubeProxyVersion - kubeletVersion = node.Status.NodeInfo.KubeletVersion - osVersion = node.Status.NodeInfo.OSImage - ) - if len(kubeVersionSpec) > 0 { - //If kube-proxy, kubelet are the same as the version to be upgraded k8s, then k8s is successfully upgraded - if kubeVersionSpec == kubeProxyVersion && kubeVersionSpec == kubeletVersion { - logrus.Infof("successfully upgraded the node: %s", node.Name) - upgradeNum++ - continue + if conditionMet(node, kubeVersionSpec, osVersionSpec) { + node.Labels[constants.LabelUpgrading] = "" + if err := r.Update(ctx, &node); err != nil { + logrus.Errorf("unable to add %s label:%v", node.Name, err) + return false, err } - } else { - if osVersionSpec == osVersion { - continue + if err := waitForUpgradeComplete(node, kubeVersionSpec, osVersionSpec); err != nil { + logrus.Errorf("failed to wait for node upgrade to complete: %v", err) + return false, err } + } else { + return false, nil } - node.Labels[constants.LabelUpgrading] = "" - if err := r.Update(ctx, &node); err != nil { - logrus.Errorf("unable to add %s label:%v", node.Name, err) + } + return true, nil +} + +func conditionMet(node corev1.Node, kubeVersionSpec string, osVersionSpec string) bool { + var ( + kubeProxyVersion = node.Status.NodeInfo.KubeProxyVersion + kubeletVersion = node.Status.NodeInfo.KubeletVersion + osVersion = node.Status.NodeInfo.OSImage + ) + if len(kubeVersionSpec) > 0 { + if kubeVersionSpec == kubeProxyVersion && kubeVersionSpec == kubeletVersion { + return false + } + } else { + if osVersionSpec == osVersion { + return false } } - if len(kubeVersionSpec) == 0 { - return true, nil + return true +} + +func waitForUpgradeComplete(node corev1.Node, kubeVersionSpec string, osVersionSpec string) error { + ctx, cancel := context.WithTimeout(context.Background(), constants.Timeout) + defer cancel() + done := make(chan struct{}) + + go func() { + wait.Until(func() { + if !conditionMet(node, kubeVersionSpec, osVersionSpec) { + close(done) + } + }, 10*time.Second, ctx.Done()) + }() + + select { + case <-done: + logrus.Infof("successful upgrade node: %s", node.Name) + case <-ctx.Done(): + // 上下文超时，跳出循环 + if ctx.Err() == context.DeadlineExceeded { + logrus.Errorf("failed to upgrade node: %s: %v", node.Name, ctx.Err()) + return ctx.Err() + } + } + return nil +} + +func min(a, b int) int { + if a < b { + return a } - return upgradeNum == len(nodeList), nil + return b } // SetupWithManager sets up the controller with the Manager. diff --git a/housekeeper/pkg/constants/constants.go b/housekeeper/pkg/constants/constants.go index 4528bda..2a2a73e 100644 --- a/housekeeper/pkg/constants/constants.go +++ b/housekeeper/pkg/constants/constants.go @@ -15,6 +15,8 @@ limitations under the License. */ package constants +import "time" + const ( // LabelUpgrading is the key of the upgrading label for nodes LabelUpgrading = "upgrade.housekeeper.io/upgrading" @@ -27,3 +29,5 @@ const ( SockDir = "/run/housekeeper-daemon" SockName = "housekeeper-daemon.sock" ) + +const Timeout = 3 * time.Minute -- Gitee From 03405862038eca0ea19019ee103e66578a36a75f Mon Sep 17 00:00:00 2001 From: duyiwei Date: Fri, 21 Jul 2023 15:31:51 +0800 Subject: [PATCH 32/38] Optimize the CERTS code and add a configuration management module --- pkg/cert/cacert.go | 63 ++++------------------- pkg/cert/certapi.go | 29 +++++++---- pkg/cert/selfsignedcert.go | 101 +++++++++++++++++++++++++++++++++++++ pkg/cert/tools.go | 62 ++++++++++++----------- 4 files changed, 163 insertions(+), 92 deletions(-) create mode 100644 pkg/cert/selfsignedcert.go diff --git a/pkg/cert/cacert.go b/pkg/cert/cacert.go index 1607ac6..0e1227e 100644 --- a/pkg/cert/cacert.go +++ b/pkg/cert/cacert.go @@ -17,64 +17,21 @@ limitations under the License. package cert import ( - "crypto/rand" - "crypto/rsa" "crypto/x509" "crypto/x509/pkix" - "math/big" - "time" - - "github.com/sirupsen/logrus" ) -func (cm *CertificateManager) GenerateCACertificate() error { - - // 生成CA的私钥和公钥 - var err error - cm.CAKey, err = rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - logrus.Errorf("Failed to generate CA privatekey: %v", err) - return err - } - caPublicKey := &cm.CACert.PublicKey - - // 生成一个介于 0 和 2^128 - 1 之间的随机序列号，并将结果存储在 serialNumber 变量中 - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - logrus.Errorf("Failed to generate serialNumber: %v", err) - return err - } - now := time.Now() - - // 设置生成证书的参数，构建CA证书模板 - caTemplate := &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{"NKD"}, - CommonName: "CA", - }, //这里还可以加很多参数信息 - NotBefore: now, - NotAfter: now.AddDate(10, 0, 0), // 有效期为10年 - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, // openssl 中的 keyUsage 字段 - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, // openssl 中的 extendedKeyUsage = clientAuth, serverAuth 字段 - BasicConstraintsValid: true, - IsCA: true, //表示用于CA - } +type RootCA struct { + SelfSignedCertKey +} - //caCertBytes是生成证书的中间步骤，它用于将证书的二进制表示存储在内存中，以便后续操作可以使用它 - caCertBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caPublicKey, cm.CAKey) - if err != nil { - logrus.Errorf("Failed to create CA Certificate(caCertBytes): %v", err) - return err - } - //cm.CACert表示已经生成的CA证书，用于存储CA证书的详细信息，例如证书序列号、主题、有效期等 - cm.CACert, err = x509.ParseCertificate(caCertBytes) - if err != nil { - logrus.Errorf("Failed to parse CA Certificate: %v", err) - return err +func (c *RootCA) Generate() error { + cfg := &CertConfig{ + Subject: pkix.Name{CommonName: "rootca", OrganizationalUnit: []string{"NestOS"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + Validity: 3650, + IsCA: true, } - logrus.Infof("Successfully generate CA certificate") - return nil + return c.SelfSignedCertKey.Generate(cfg, "rootca") } diff --git a/pkg/cert/certapi.go b/pkg/cert/certapi.go index 4170ed6..21304f1 100644 --- a/pkg/cert/certapi.go +++ b/pkg/cert/certapi.go @@ -17,21 +17,30 @@ limitations under the License. package cert import ( - "crypto/rsa" "crypto/x509" + "crypto/x509/pkix" + "net" + "time" ) type CertificateGenerator interface { GenerateCACertificate() error - GenerateCertificate(commonName string) error + GenerateSignedCertificate(commonName string) error } -// CertificateManager 是证书管理器 -type CertificateManager struct { - CAKey *rsa.PrivateKey - CACert *x509.Certificate - ComponentKey *rsa.PrivateKey - ComponentCert *x509.Certificate - ValidDays int - CertDirectory string +// CertKey 包含证书和私钥 +type CertKey struct { + CertRaw []byte + KeyRaw []byte + SavePath string +} + +type CertConfig struct { + DNSNames []string + ExtKeyUsages []x509.ExtKeyUsage + IPAddresses []net.IP + KeyUsages x509.KeyUsage + Subject pkix.Name + Validity time.Duration + IsCA bool } diff --git a/pkg/cert/selfsignedcert.go b/pkg/cert/selfsignedcert.go new file mode 100644 index 0000000..91f9d4c --- /dev/null +++ b/pkg/cert/selfsignedcert.go @@ -0,0 +1,101 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "math/big" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// SelfSignedCertificate 只负责创建自签名的证书，这里只传入cfg和privatekey +func SelfSignedCertificate(cfg *CertConfig, key *rsa.PrivateKey) (*x509.Certificate, error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + logrus.Errorf("Failed to generate serialNumber: %v", err) + return nil, err + } + certTemplate := x509.Certificate{ + BasicConstraintsValid: true, + IsCA: cfg.IsCA, + KeyUsage: cfg.KeyUsages, + NotAfter: time.Now().Add(cfg.Validity), + NotBefore: time.Now(), + SerialNumber: serialNumber, + Subject: cfg.Subject, + } + // 判断subject字段中CommonName和OrganizationalUnit是否为空 + if len(cfg.Subject.CommonName) == 0 || len(cfg.Subject.OrganizationalUnit) == 0 { + return nil, errors.Errorf("certification's subject is not set, or invalid") + } + + //certBytes是生成证书的中间步骤，它用于将证书的二进制表示存储在内存中，以便后续操作可以使用它 + certBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, key.Public(), key) + if err != nil { + return nil, errors.Wrap(err, "failed to create certificate") + } + + return x509.ParseCertificate(certBytes) +} + +/*GenerateSelfSignedCertificate负责根据cfg生成私钥和证书 + 在这一步会调用PrivateKey生成私钥并调用SelfSignedCertificate生成证书，并将结果返回*/ +func GenerateSelfSignedCertificate(cfg *CertConfig) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := PrivateKey() + if err != nil { + logrus.Debugf("Failed to generate private key: %s", err) + return nil, nil, errors.Wrap(err, "Failed to generate private key") + } + + //这里的crt是parse之后的，表示已经生成的CA证书，用于存储CA证书的详细信息，例如证书序列号、主题、有效期等 + crt, err := SelfSignedCertificate(cfg, key) + if err != nil { + logrus.Debugf("Failed to create self-signed certificate: %s", err) + return nil, nil, errors.Wrap(err, "failed to create self-signed certificate") + } + return key, crt, nil +} + +type SelfSignedCertKey struct { + CertKey +} + +//自签名证书生成器，封装后该方法用于所有自签名的证书,并将证书和私钥转换格式后保存 +func (c *SelfSignedCertKey) Generate(cfg *CertConfig, filename string) error { + + key, crt, err := GenerateSelfSignedCertificate(cfg) + if err != nil { + return errors.Wrap(err, "Failed to generate self-signed cert/key pair") + } + + c.KeyRaw = PrivateKeyToPem(key) + c.CertRaw = CertToPem(crt) + + err = c.SaveCertificateToFile(filename) + if err != nil { + logrus.Errorf("Faile to save %s: %v", filename, err) + } + + return nil + +} diff --git a/pkg/cert/tools.go b/pkg/cert/tools.go index e63826c..21a8a7c 100644 --- a/pkg/cert/tools.go +++ b/pkg/cert/tools.go @@ -17,48 +17,52 @@ limitations under the License. package cert import ( + "crypto/rand" + "crypto/rsa" "crypto/x509" "encoding/pem" "io/ioutil" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -// CACertPEM 返回CA证书的PEM格式字节切片 -func (cm *CertificateManager) CACertPEM() []byte { - return pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: cm.CACert.Raw, - }) -} +//PrivateKey负责生成密钥 +func PrivateKey() (*rsa.PrivateKey, error) { + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, errors.Wrap(err, "Failed to generate RSA private key") + } -// CAKeyPEM 返回CA私钥的PEM格式字节切片 -func (cm *CertificateManager) CAKeyPEM() []byte { - return pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(cm.CAKey), - }) + return rsaKey, nil } -// ComponentCertPEM 返回组件证书的PEM格式字节切片 -func (cm *CertificateManager) ComponentCertPEM() []byte { - return pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: cm.ComponentCert.Raw, - }) +// PrivateKeyToPem 返回私钥的PEM格式字节切片 +func PrivateKeyToPem(key *rsa.PrivateKey) []byte { + keyInBytes := x509.MarshalPKCS1PrivateKey(key) + keyinPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: keyInBytes, + }, + ) + return keyinPem } -// ComponentKeyPEM 返回组件私钥的PEM格式字节切片 -func (cm *CertificateManager) ComponentKeyPEM() []byte { - return pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(cm.ComponentKey), - }) +// CACertPEM 返回证书的PEM格式字节切片 +func CertToPem(cert *x509.Certificate) []byte { + certInPem := pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Raw, + }, + ) + return certInPem } // SaveCertificateToFile 将证书保存到文件 -func (cm *CertificateManager) SaveCertificateToFile(filename string, certPEM []byte) error { - err := ioutil.WriteFile(cm.CertDirectory+"/"+filename, certPEM, 0644) +func (c *CertKey) SaveCertificateToFile(filename string) error { + err := ioutil.WriteFile(c.SavePath+"/"+filename, c.CertRaw, 0644) if err != nil { logrus.Errorf("Faile to save %s: %v", filename, err) return err @@ -70,8 +74,8 @@ func (cm *CertificateManager) SaveCertificateToFile(filename string, certPEM []b } // SavePrivateKeyToFile 将私钥保存到文件 -func (cm *CertificateManager) SavePrivateKeyToFile(filename string, keyPEM []byte) error { - err := ioutil.WriteFile(cm.CertDirectory+"/"+filename, keyPEM, 0600) +func (c *CertKey) SavePrivateKeyToFile(filename string) error { + err := ioutil.WriteFile(c.SavePath+"/"+filename, c.KeyRaw, 0600) if err != nil { logrus.Errorf("Faile to save %s: %v", filename, err) return err -- Gitee From d4bf72cf447a263282af1252d5ed0278cfce9b5c Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Fri, 21 Jul 2023 15:18:51 +0800 Subject: [PATCH 33/38] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6json=E8=BD=ACtf=E6=A0=BC=E5=BC=8F=E6=96=B9?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/json/main.tf.json | 25 ++++++++ data/templates/openstack/main.tf.template | 56 ++++++++++++++++++ pkg/infra/assets/assets.go | 15 ----- pkg/infra/assets/cluster/cluster.go | 8 +-- pkg/infra/assets/cluster/tfvars.go | 58 ++++++++++++++++++- pkg/infra/infradeployer.go | 28 --------- pkg/infra/initconfig.go | 25 -------- pkg/infra/openstack.go | 29 ---------- .../openstack/tfvars.go} | 41 ++++++++----- 9 files changed, 168 insertions(+), 117 deletions(-) create mode 100644 data/json/main.tf.json create mode 100644 data/templates/openstack/main.tf.template delete mode 100755 pkg/infra/infradeployer.go delete mode 100755 pkg/infra/initconfig.go delete mode 100755 pkg/infra/openstack.go rename pkg/infra/{factory.go => tfvars/openstack/tfvars.go} (42%) mode change 100755 => 100644 diff --git a/data/json/main.tf.json b/data/json/main.tf.json new file mode 100644 index 0000000..927d0b7 --- /dev/null +++ b/data/json/main.tf.json @@ -0,0 +1,25 @@ +{ + "openstack": { + "user_name": "XXXXXXXXXX", + "password": "XXXXXXXXXX", + "tenant_name": "XXXXXXXXXX", + "auth_url": "XXXXXXXXXX", + "region": "XXXXXXXXXX" + }, + "flavor": { + "name": "XXXXXXXXXX", + "ram": "XXXXXXXXXX", + "vcpus": "XXXXXXXXXX", + "disk": "XXXXXXXXXX", + "is_public": "true" + }, + "instance": { + "count": "XXXXXXXXXX", + "name": "XXXXXXXXXX", + "image_name": "XXXXXXXXXX", + "key_pair": "XXXXXXXXXX" + }, + "floatip": { + "pool": "XXXXXXXXXX" + } +} \ No newline at end of file diff --git a/data/templates/openstack/main.tf.template b/data/templates/openstack/main.tf.template new file mode 100644 index 0000000..20583d2 --- /dev/null +++ b/data/templates/openstack/main.tf.template @@ -0,0 +1,56 @@ +terraform { + required_version = ">= 0.14.0" + required_providers { + openstack = { + source = "XXXXXXXXXX" + } + } +} + +provider "openstack" { + user_name = "{{.Openstack.User_name}}" + password = "{{.Openstack.Password}}" + tenant_name = "{{.Openstack.Tenant_name}}" + auth_url = "{{.Openstack.Auth_url}}" + region = "{{.Openstack.Region}}" +} + +resource "openstack_compute_flavor_v2" "flavor" { + name = "{{.Flavor.Name}}" + ram = "{{.Flavor.Ram}}" + vcpus = "{{.Flavor.Vcpus}}" + disk = "{{.Flavor.Disk}}" + is_public = "{{.Flavor.Is_public}}" +} + +resource "openstack_compute_instance_v2" "instance" { + count = "{{.Instance.Count}}" + name = "{{.Instance.Name}}" + image_name = "{{.Instance.Image_name}}" + flavor_name = openstack_compute_flavor_v2.flavor.name + key_pair = "{{.Instance.Key_pair}}" + security_groups = ["XXXXXXXXXX"] + user_data = "${file("XXXXXXXXXX")}" + + network { + name = "XXXXXXXXXX" + } +} + +resource "openstack_networking_floatingip_v2" "floatip" { + count = length(openstack_compute_instance_v2.instance) + pool = "{{.Floatip.Pool}}" +} + +resource "openstack_compute_floatingip_associate_v2" "floatip" { + count = length(openstack_compute_instance_v2.instance) + instance_id = openstack_compute_instance_v2.instance.*.id[count.index] + floating_ip = openstack_networking_floatingip_v2.floatip.*.address[count.index] +} + +output "instance_info" { + value = { + floating_ip_addresses = openstack_networking_floatingip_v2.floatip.*.address + instance_status = openstack_compute_instance_v2.instance.*.power_state + } +} diff --git a/pkg/infra/assets/assets.go b/pkg/infra/assets/assets.go index 75fdc8d..81fa8cd 100755 --- a/pkg/infra/assets/assets.go +++ b/pkg/infra/assets/assets.go @@ -16,21 +16,6 @@ limitations under the License. package assets -// path : contents -type Assets map[string][]byte - -func (a Assets) ToDir(dirname string) error { - return nil -} - -func (a *Assets) Merge(b Assets) *Assets { - return a -} - -type AssetsGenerator interface { - GenerateAssets() Assets -} - type File struct { Filename string Data []byte diff --git a/pkg/infra/assets/cluster/cluster.go b/pkg/infra/assets/cluster/cluster.go index 2fe19e5..5b38e6a 100644 --- a/pkg/infra/assets/cluster/cluster.go +++ b/pkg/infra/assets/cluster/cluster.go @@ -111,11 +111,11 @@ func (c *Cluster) Destroy() error { logrus.Infof("start to destroy %s in %s", c.Name, c.Platform) - // TODO: Destroy的tfvarsFiles的获取 + // Question: Destroy的tfvarsFiles的获取 - // terraformVariables := &TerraformVariables{} - // tfvarsFiles := make([]*assets.File, 0, len(terraformVariables.Files())+len(c.Platform)+len(c.Name)) - // tfvarsFiles = append(tfvarsFiles, terraformVariables.Files()...) + terraformVariables := &TerraformVariables{} + tfvarsFiles := make([]*assets.File, 0, len(terraformVariables.Files())+len(c.Platform)+len(c.Name)) + tfvarsFiles = append(tfvarsFiles, terraformVariables.Files()...) err := c.destroyStage(dir, terraformDir, tfvarsFiles) if err != nil { diff --git a/pkg/infra/assets/cluster/tfvars.go b/pkg/infra/assets/cluster/tfvars.go index 6d5729f..a2f1b17 100644 --- a/pkg/infra/assets/cluster/tfvars.go +++ b/pkg/infra/assets/cluster/tfvars.go @@ -16,7 +16,16 @@ limitations under the License. package cluster -import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/assets" +import ( + "encoding/json" + "html/template" + "os" + "path/filepath" + + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/assets" + "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/infra/tfvars/openstack" + "github.com/pkg/errors" +) type TerraformVariables struct { FileList []*assets.File @@ -25,3 +34,50 @@ type TerraformVariables struct { func (t *TerraformVariables) Files() []*assets.File { return t.FileList } + +func Generate() error { + // 从文件中读取 jsonData + jsonData, err := os.ReadFile("data/json/openstack/main.tf.json") + if err != nil { + return errors.Wrap(err, "error reading json data") + } + + // 解析 JSON 数据 + var terraformData openstack.TerraformData + err = json.Unmarshal(jsonData, &terraformData) + if err != nil { + return errors.Wrap(err, "error parsing json data") + } + + // 从文件中读取 terraformConfig + terraformConfig, err := os.ReadFile("data/templates/openstack/main.tf.template") + if err != nil { + return errors.Wrap(err, "error reading terraform config template") + } + + // 使用模板填充数据 + tmpl, err := template.New("terraform").Parse(string(terraformConfig)) + if err != nil { + return errors.Wrap(err, "error creating terraform config template") + } + + // 创建一个新的文件用于写入填充后的数据 + tfDir := filepath.Join("/root", "terraform") + if err := os.MkdirAll(tfDir, os.ModePerm); err != nil { + return errors.Wrap(err, "could not create the terraform directory") + } + + outputFile, err := os.Create(filepath.Join(tfDir, "main.tf")) + if err != nil { + return errors.Wrap(err, "error creating terraform config") + } + defer outputFile.Close() + + // 将填充后的数据写入文件 + err = tmpl.Execute(outputFile, terraformData) + if err != nil { + return errors.Wrap(err, "error executing terraform config") + } + + return nil +} diff --git a/pkg/infra/infradeployer.go b/pkg/infra/infradeployer.go deleted file mode 100755 index dc28150..0000000 --- a/pkg/infra/infradeployer.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2023 KylinSoft Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package infra - -type InfraSpec struct { - diskSize string - memorySize string - image string - clusterCIDR string - initFileContent string -} - -type InfraDeployer interface { -} diff --git a/pkg/infra/initconfig.go b/pkg/infra/initconfig.go deleted file mode 100755 index 288c3b3..0000000 --- a/pkg/infra/initconfig.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2023 KylinSoft Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package infra - -type InitConfig struct { - OsType string -} - -type BootConfigAssembler interface { - Assemble(assets Assets) InitConfig -} diff --git a/pkg/infra/openstack.go b/pkg/infra/openstack.go deleted file mode 100755 index 76d4232..0000000 --- a/pkg/infra/openstack.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2023 KylinSoft Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package infra - -type OpenstackDeployer struct { -} - -func (t OpenstackDeployer) Create() error { - return nil -} - -func (t OpenstackDeployer) Destroy() error { - - return nil -} diff --git a/pkg/infra/factory.go b/pkg/infra/tfvars/openstack/tfvars.go old mode 100755 new mode 100644 similarity index 42% rename from pkg/infra/factory.go rename to pkg/infra/tfvars/openstack/tfvars.go index 3db63de..693bee2 --- a/pkg/infra/factory.go +++ b/pkg/infra/tfvars/openstack/tfvars.go @@ -14,20 +14,31 @@ See the License for the specific language governing permissions and limitations under the License. */ -package infra +package openstack -import "gitee.com/openeuler/nestos-kubernetes-deployer/pkg/bootconfig" - -func GetInfraDeployer(driverType string) InfraDeployer { - if driverType == "openstack" { - return OpenstackDeployer{} - } - return nil -} - -func GetBootConfigAssembler(osType string) BootConfigAssembler { - if osType == "NestOS" { - return bootconfig.IgnitionAssembler{} - } - return nil +// 定义 JSON 数据的结构 +type TerraformData struct { + Openstack struct { + User_name string `json:"user_name"` + Password string `json:"password"` + Tenant_name string `json:"tenant_name"` + Auth_url string `json:"auth_url"` + Region string `json:"region"` + } `json:"openstack"` + Flavor struct { + Name string `json:"name"` + Ram string `json:"ram"` + Vcpus string `json:"vcpus"` + Disk string `json:"disk"` + Is_public string `json:"is_public"` + } `json:"flavor"` + Instance struct { + Count string `json:"count"` + Name string `json:"name"` + Image_name string `json:"image_name"` + Key_pair string `json:"key_pair"` + } `json:"instance"` + Floatip struct { + Pool string `json:"pool"` + } `json:"floatip"` } -- Gitee From cb5a64f0aeb8cecae1b31bab2a3dda64bbb02d53 Mon Sep 17 00:00:00 2001 From: lauk Date: Fri, 21 Jul 2023 17:39:28 +0800 Subject: [PATCH 34/38] Add the create CR resource code to the upgrade command --- cmd/nkd/upgrade.go | 99 ++++++++++- go.mod | 24 +++ go.sum | 416 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 538 insertions(+), 1 deletion(-) diff --git a/cmd/nkd/upgrade.go b/cmd/nkd/upgrade.go index a2b86b5..5550b32 100755 --- a/cmd/nkd/upgrade.go +++ b/cmd/nkd/upgrade.go @@ -1,7 +1,22 @@ package main import ( + "context" + "fmt" + "time" + + "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/yaml" + + wait "k8s.io/apimachinery/pkg/util/wait" ) func newUpgradeCmd() *cobra.Command { @@ -15,6 +30,88 @@ func newUpgradeCmd() *cobra.Command { } func runUpgradeCmd(command *cobra.Command, args []string) error { - //todo: Upgrade k8s version function implementation + var ( + osVersion = "" + osImageURL = "" + kubeVersion = "" + evictPodForce = false + maxUnavailable = 2 + loopTimeout = 2 * time.Minute + ) + // Get the kubeconfig configuration + config, err := clientcmd.BuildConfigFromFlags("", "/root/.kube/config") + if err != nil { + config, err = rest.InClusterConfig() + if err != nil { + logrus.Errorf("Error getting Kubernetes client config: %v\n", err) + return err + } + } + + // Create dynamic client + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + logrus.Errorf("Error creating Dynamic client: %v\n", err) + return err + } + + // Define the YAML data for the Custom Resource (CR) + yamlData := fmt.Sprintf(` +apiVersion: housekeeper.io/v1alpha1 +kind: Update +metadata: + name: update-sample + namespace: housekeeper-system +spec: + osVersion: %s + osImageURL: %s + kubeVersion: %s + evictPodForce: %t + maxUnavailable: %d +`, osVersion, osImageURL, kubeVersion, evictPodForce, maxUnavailable) + + var unstructuredObj unstructured.Unstructured + err = yaml.Unmarshal([]byte(yamlData), &unstructuredObj) + if err != nil { + logrus.Errorf("Error unmarshalling YAML: %v\n", err) + return err + } + + // Create or Update CR + resource := schema.GroupVersionResource{ + Group: "housekeeper.io", + Version: "v1alpha1", + Resource: "updates", // Pluralized resource name + } + + // The loop attempts to create or update a CR until it succeeds or times out + if err := wait.PollImmediate(2*time.Second, loopTimeout, func() (bool, error) { + gvk := unstructuredObj.GroupVersionKind() + dynamicResource := dynamicClient.Resource(gvk.GroupVersion().WithResource(resource.Resource)).Namespace(unstructuredObj.GetNamespace()) + + //Attempts to get the specified Custom Resource from the Kubernetes API Server. + obj, err := dynamicResource.Get(context.Background(), unstructuredObj.GetName(), metav1.GetOptions{}) + if err != nil { + // Not found, create the resource + _, err = dynamicResource.Create(context.Background(), &unstructuredObj, metav1.CreateOptions{}) + if err == nil { + logrus.Infof("Custom Resource created successfully!") + return true, nil + } + } else { + // Found, update the resource + unstructuredObj.SetResourceVersion(obj.GetResourceVersion()) + _, err = dynamicResource.Update(context.Background(), &unstructuredObj, metav1.UpdateOptions{}) + if err == nil { + logrus.Infof("Custom Resource updated successfully!") + return true, nil + } + } + logrus.Errorf("Error creating or updating CR: %v\n", err) + return false, nil + }); err != nil { + logrus.Errorf("Timeout while waiting for Custom Resource to be created or updated.") + } + return nil } diff --git a/go.mod b/go.mod index a06340c..4df5559 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,36 @@ require ( require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gofuzz v1.1.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/terraform-json v0.16.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/zclconf/go-cty v1.13.1 // indirect golang.org/x/crypto v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apimachinery v0.27.4 // indirect + k8s.io/client-go v0.27.4 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 4738bb3..5062ad1 100644 --- a/go.sum +++ b/go.sum @@ -1,39 +1,164 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hc-install v0.5.0 h1:D9bl4KayIYKEeJ4vUDe9L5huqxZXczKaykSRcmQ0xY0= github.com/hashicorp/terraform-exec v0.18.1 h1:LAbfDvNQU1l0NOQlTuudjczVhHj061fNX5H8XZxHlH4= github.com/hashicorp/terraform-exec v0.18.1/go.mod h1:58wg4IeuAJ6LVsLUeD2DWZZoc/bYi6dzhLHzxM41980= github.com/hashicorp/terraform-json v0.16.0 h1:UKkeWRWb23do5LNAFlh/K3N0ymn1qTOO8c+85Albo3s= github.com/hashicorp/terraform-json v0.16.0/go.mod h1:v0Ufk9jJnk6tcIZvScHvetlKfiNTC+WS21mnXIlc0B0= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/openshift/installer v0.16.1 h1:PmjALN9x1NVNVi3SCqfz0ZwVCgOkQLQWo2nHYXREq/A= github.com/openshift/installer v0.16.1/go.mod h1:VWGgpJgF8DGCKQjbccnigglhZnHtRLCZ6cxqkXN4Ck0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -43,22 +168,313 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zclconf/go-cty v1.13.1 h1:0a6bRwuiSHtAmqCqNOE+c2oHgepv0ctoxU4FUe43kwc= github.com/zclconf/go-cty v1.13.1/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs= +k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/client-go v0.27.4 h1:vj2YTtSJ6J4KxaC88P4pMPEQECWMY8gqPqsTgUKzvjk= +k8s.io/client-go v0.27.4/go.mod h1:ragcly7lUlN0SRPk5/ZkGnDjPknzb37TICq07WhI6Xc= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -- Gitee From 8102cdc7f9709f591a46ef2e8a747fc36811f2d7 Mon Sep 17 00:00:00 2001 From: jianli-97 Date: Tue, 25 Jul 2023 10:43:40 +0800 Subject: [PATCH 35/38] =?UTF-8?q?=E8=A1=A5=E5=85=85terraform=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=A8=A1=E6=9D=BF=E6=96=87=E4=BB=B6=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/json/main.tf.json | 25 ---------- data/templates/openstack/main.tf.template | 61 +++++++++++++++++++---- 2 files changed, 50 insertions(+), 36 deletions(-) delete mode 100644 data/json/main.tf.json diff --git a/data/json/main.tf.json b/data/json/main.tf.json deleted file mode 100644 index 927d0b7..0000000 --- a/data/json/main.tf.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "openstack": { - "user_name": "XXXXXXXXXX", - "password": "XXXXXXXXXX", - "tenant_name": "XXXXXXXXXX", - "auth_url": "XXXXXXXXXX", - "region": "XXXXXXXXXX" - }, - "flavor": { - "name": "XXXXXXXXXX", - "ram": "XXXXXXXXXX", - "vcpus": "XXXXXXXXXX", - "disk": "XXXXXXXXXX", - "is_public": "true" - }, - "instance": { - "count": "XXXXXXXXXX", - "name": "XXXXXXXXXX", - "image_name": "XXXXXXXXXX", - "key_pair": "XXXXXXXXXX" - }, - "floatip": { - "pool": "XXXXXXXXXX" - } -} \ No newline at end of file diff --git a/data/templates/openstack/main.tf.template b/data/templates/openstack/main.tf.template index 20583d2..ec69fa5 100644 --- a/data/templates/openstack/main.tf.template +++ b/data/templates/openstack/main.tf.template @@ -1,8 +1,8 @@ terraform { - required_version = ">= 0.14.0" required_providers { openstack = { - source = "XXXXXXXXXX" + source = "terraform-provider-openstack/openstack" + version = "1.52.1" } } } @@ -15,6 +15,20 @@ provider "openstack" { region = "{{.Openstack.Region}}" } +resource "openstack_images_image_v2" "image" { + name = "{{.Image.Name}}" + + image_source_url = "{{.Image.Image_source_url}}" + image_source_username = "{{.Image.Image_source_username}}" + image_source_password = "{{.Image.Image_source_password}}" + web_download = "{{.Image.Web_download}}" + + local_file_path = "{{.Image.local_file_path}}" + + container_format = "{{.Image.Container_format}}" + disk_format = "{{.Image.Disk_format}}" +} + resource "openstack_compute_flavor_v2" "flavor" { name = "{{.Flavor.Name}}" ram = "{{.Flavor.Ram}}" @@ -23,17 +37,41 @@ resource "openstack_compute_flavor_v2" "flavor" { is_public = "{{.Flavor.Is_public}}" } +resource "openstack_compute_keypair_v2" "keypair" { + name = "{{.Keypair.Name}}" + public_key = "{{.Keypair.Public_key}}" +} + +resource "openstack_compute_secgroup_v2" "secgroup" { + name = "{{.Secgroup.Name}}" + description = "{{.Secgroup.Name}}" + + rule { + from_port = 22 + to_port = 22 + ip_protocol = "tcp" + cidr = "0.0.0.0/0" + } + + rule { + from_port = -1 + to_port = -1 + ip_protocol = "icmp" + cidr = "0.0.0.0/0" + } +} + resource "openstack_compute_instance_v2" "instance" { count = "{{.Instance.Count}}" name = "{{.Instance.Name}}" - image_name = "{{.Instance.Image_name}}" + image_name = openstack_images_image_v2.image.name flavor_name = openstack_compute_flavor_v2.flavor.name - key_pair = "{{.Instance.Key_pair}}" - security_groups = ["XXXXXXXXXX"] - user_data = "${file("XXXXXXXXXX")}" + key_pair = openstack_compute_keypair_v2.keypair.name + security_groups = [openstack_compute_secgroup_v2.secgroup.name] + user_data = "{{.Instance.User_data}}" network { - name = "XXXXXXXXXX" + name = "{{.Instance.Network.Name}}" } } @@ -42,15 +80,16 @@ resource "openstack_networking_floatingip_v2" "floatip" { pool = "{{.Floatip.Pool}}" } -resource "openstack_compute_floatingip_associate_v2" "floatip" { - count = length(openstack_compute_instance_v2.instance) - instance_id = openstack_compute_instance_v2.instance.*.id[count.index] +resource "openstack_compute_floatingip_associate_v2" "fip_associate" { + count = length(openstack_compute_instance_v2.instance) floating_ip = openstack_networking_floatingip_v2.floatip.*.address[count.index] + instance_id = openstack_compute_instance_v2.instance.*.id[count.index] } output "instance_info" { value = { - floating_ip_addresses = openstack_networking_floatingip_v2.floatip.*.address instance_status = openstack_compute_instance_v2.instance.*.power_state + access_ip_v4 = openstack_compute_instance_v2.instance.*.access_ip_v4 + floating_ip = openstack_networking_floatingip_v2.floatip.*.address } } -- Gitee From 9643380578bb60ac3d6ea31b6e58897d04e6073a Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 26 Jul 2023 17:28:14 +0800 Subject: [PATCH 36/38] housekeeper-operator code optimization --- cmd/nkd/upgrade.go | 3 +- housekeeper/Makefile | 4 +- .../controllers/update_controller.go | 198 +++++++++--------- housekeeper/pkg/constants/constants.go | 7 +- 4 files changed, 113 insertions(+), 99 deletions(-) diff --git a/cmd/nkd/upgrade.go b/cmd/nkd/upgrade.go index 5550b32..8493cac 100755 --- a/cmd/nkd/upgrade.go +++ b/cmd/nkd/upgrade.go @@ -37,9 +37,10 @@ func runUpgradeCmd(command *cobra.Command, args []string) error { evictPodForce = false maxUnavailable = 2 loopTimeout = 2 * time.Minute + kubeconfig = "" ) // Get the kubeconfig configuration - config, err := clientcmd.BuildConfigFromFlags("", "/root/.kube/config") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { config, err = rest.InClusterConfig() if err != nil { diff --git a/housekeeper/Makefile b/housekeeper/Makefile index 590e7f6..6972971 100644 --- a/housekeeper/Makefile +++ b/housekeeper/Makefile @@ -35,12 +35,14 @@ SHELL = /usr/bin/env bash -o pipefail ##@ Build .PHONY: all -all: housekeeper-operator-manager housekeeper-controller-manager +all: housekeeper-operator-manager housekeeper-controller-manager housekeeper-daemon # Build binary housekeeper-operator-manager: go build -o bin/housekeeper-operator-manager operator/housekeeper-operator/main.go housekeeper-controller-manager: go build -o bin/housekeeper-controller-manager operator/housekeeper-controller/main.go +housekeeper-daemon: + go build -o bin/housekeeper-daemon daemon/main.go # Build the docker image .PHONY: docker-build diff --git a/housekeeper/operator/housekeeper-operator/controllers/update_controller.go b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go index 0cc7af8..266bc61 100644 --- a/housekeeper/operator/housekeeper-operator/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-operator/controllers/update_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "sync" "time" "github.com/sirupsen/logrus" @@ -29,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,91 +57,95 @@ type UpdateReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) - if r.Client == nil { return common.NoRequeue, nil } + var crMutex sync.Mutex + crMutex.Lock() + defer crMutex.Unlock() ctx = context.Background() - //返回worker节点的数量 - upInstance, nodesNum, err := getUpgradeInstance(ctx, r, req.NamespacedName) + return reconcile(ctx, r, req) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&housekeeperiov1alpha1.Update{}). + Complete(r) +} + +func reconcile(ctx context.Context, r common.ReadWriterClient, req ctrl.Request) (ctrl.Result, error) { + var update housekeeperiov1alpha1.Update + if err := r.Get(ctx, req.NamespacedName, &update); err != nil { + logrus.Errorf("unable to fetch update instance: %v", err) + return common.NoRequeue, err + } + if len(update.Spec.OSVersion) == 0 { + logrus.Warning("os version is required") + return common.RequeueAfter, nil + } + masterNodesItems, err := getMasterNodesItems(ctx, r) if err != nil { return common.RequeueNow, err } - limit := min(upInstance.Spec.MaxUnavailable, nodesNum) - if requeueAfter, err := setLabels(ctx, r, req, limit, upInstance); err != nil { - logrus.Errorf("unable set nodes label: %v", err) + workerNodesItems, err := getWorkerNodesItems(ctx, r) + if err != nil { return common.RequeueNow, err - } else if requeueAfter { - return common.RequeueAfter, nil } - return common.RequeueNow, nil + if assignUpdated(ctx, r, masterNodesItems, 1, update); err != nil { + return common.RequeueNow, err + } + maxUnavailable := min(update.Spec.MaxUnavailable, len(workerNodesItems)) + if assignUpdated(ctx, r, masterNodesItems, maxUnavailable, update); err != nil { + return common.RequeueNow, err + } + + return common.NoRequeue, nil } -func getUpgradeInstance(ctx context.Context, r common.ReadWriterClient, name types.NamespacedName) ( - upInstance housekeeperiov1alpha1.Update, nodeNum int, err error) { - if err = r.Get(ctx, name, &upInstance); err != nil { - logrus.Errorf("unable to fetch upgrade instance: %v", err) +func getMasterNodesItems(ctx context.Context, r common.ReadWriterClient) ( + nodesItems []corev1.Node, err error) { + reqUpgrade, err := labels.NewRequirement(constants.LabelUpgrading, selection.DoesNotExist, nil) + if err != nil { + logrus.Errorf("unable to create requirement %s: %v", reqUpgrade, err) return } - requirement, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) + reqMaster, err := labels.NewRequirement(constants.LabelMaster, selection.Exists, nil) if err != nil { - logrus.Errorf("unable to create requirement %s: %v"+constants.LabelMaster, err) + logrus.Errorf("unable to create requirement %s: %v", constants.LabelMaster, err) return } - nodesItems, err := getNodes(ctx, r, 0, *requirement) + nodesItems, err = getNodes(ctx, r, *reqUpgrade, *reqMaster) if err != nil { - logrus.Errorf("failed to get nodes list: %v", err) + logrus.Errorf("failed to get master nodes list: %v", err) return } - nodeNum = len(nodesItems) return } -func setLabels(ctx context.Context, r common.ReadWriterClient, req ctrl.Request, limit int, - upInstance housekeeperiov1alpha1.Update) (bool, error) { +func getWorkerNodesItems(ctx context.Context, r common.ReadWriterClient) ( + nodesItems []corev1.Node, err error) { reqUpgrade, err := labels.NewRequirement(constants.LabelUpgrading, selection.DoesNotExist, nil) if err != nil { - logrus.Errorf("unable to create upgrade label requirement: %v", err) - return false, err - } - reqMaster, err := labels.NewRequirement(constants.LabelMaster, selection.Exists, nil) - if err != nil { - logrus.Errorf("unable to create master label requirement: %v", err) - return false, err - } - reqNoMaster, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) - if err != nil { - logrus.Errorf("unable to create non-master label requirement: %v", err) - return false, err - } - masterNodes, err := getNodes(ctx, r, 1, *reqUpgrade, *reqMaster) - if err != nil { - logrus.Errorf("unable to get master node list: %v", err) - return false, err + logrus.Errorf("unable to create requirement %s: %v", reqUpgrade, err) + return } - //limit: 限制worker节点每次升级的数量 - noMasterNodes, err := getNodes(ctx, r, limit, *reqUpgrade, *reqNoMaster) + reqWorker, err := labels.NewRequirement(constants.LabelMaster, selection.DoesNotExist, nil) if err != nil { - logrus.Errorf("unable to get non-master node list: %v", err) - return false, err + logrus.Errorf("unable to create requirement %s: %v"+constants.LabelMaster, err) + return } - needRequeue, err := assignUpdated(ctx, r, masterNodes, upInstance) + nodesItems, err = getNodes(ctx, r, *reqUpgrade, *reqWorker) if err != nil { - logrus.Errorf("unabel to add upgrade label of the master nodes: %v", err) - return false, err - } else if needRequeue { - return true, nil - } - if needRequeue, err = assignUpdated(ctx, r, noMasterNodes, upInstance); err != nil { - logrus.Errorf("unabel to add upgrade label of non-master nodes: %v", err) - return false, err + logrus.Errorf("failed to get worker nodes list: %v", err) + return } - return needRequeue, nil + return } -func getNodes(ctx context.Context, r common.ReadWriterClient, limit int, reqs ...labels.Requirement) ([]corev1.Node, error) { +func getNodes(ctx context.Context, r common.ReadWriterClient, reqs ...labels.Requirement) ([]corev1.Node, error) { var nodeList corev1.NodeList - opts := client.ListOptions{LabelSelector: labels.NewSelector().Add(reqs...), Limit: int64(limit)} + opts := client.ListOptions{LabelSelector: labels.NewSelector().Add(reqs...)} if err := r.List(ctx, &nodeList, &opts); err != nil { logrus.Errorf("unable to list nodes with requirements: %v", err) return nil, err @@ -151,56 +155,55 @@ func getNodes(ctx context.Context, r common.ReadWriterClient, limit int, reqs .. // Add the label to nodes func assignUpdated(ctx context.Context, r common.ReadWriterClient, nodeList []corev1.Node, - upInstance housekeeperiov1alpha1.Update) (bool, error) { + maxUnavailable int, upInstance housekeeperiov1alpha1.Update) error { var ( kubeVersionSpec = upInstance.Spec.KubeVersion osVersionSpec = upInstance.Spec.OSVersion + count = 0 + wg sync.WaitGroup ) - if len(osVersionSpec) == 0 { - logrus.Warning("os version is required") - return false, nil - } - if len(nodeList) == 0 { - return false, nil - } + + // 创建一个通道来接收任务结果 + resultChan := make(chan error) + for _, node := range nodeList { + if count >= maxUnavailable { + count = 0 + //为了控制升级任务的并发数，每处理 maxUnavailable 个节点后，休眠 2 分钟 + time.Sleep(constants.NodeSleepTime) + } if conditionMet(node, kubeVersionSpec, osVersionSpec) { node.Labels[constants.LabelUpgrading] = "" if err := r.Update(ctx, &node); err != nil { logrus.Errorf("unable to add %s label:%v", node.Name, err) - return false, err + return err } - if err := waitForUpgradeComplete(node, kubeVersionSpec, osVersionSpec); err != nil { - logrus.Errorf("failed to wait for node upgrade to complete: %v", err) - return false, err - } - } else { - return false, nil + count++ + wg.Add(1) // 增加 WaitGroup 的计数器 + go func(node corev1.Node) { + waitForUpgradeComplete(node, kubeVersionSpec, osVersionSpec, resultChan, &wg) + }(node) } } - return true, nil -} + //等待所有任务完成 + wg.Wait() -func conditionMet(node corev1.Node, kubeVersionSpec string, osVersionSpec string) bool { - var ( - kubeProxyVersion = node.Status.NodeInfo.KubeProxyVersion - kubeletVersion = node.Status.NodeInfo.KubeletVersion - osVersion = node.Status.NodeInfo.OSImage - ) - if len(kubeVersionSpec) > 0 { - if kubeVersionSpec == kubeProxyVersion && kubeVersionSpec == kubeletVersion { - return false - } - } else { - if osVersionSpec == osVersion { - return false + //关闭结果通道 + close(resultChan) + // 遍历结果通道，处理每个任务的结果 + for err := range resultChan { + if err != nil { + return err } } - return true + return nil } -func waitForUpgradeComplete(node corev1.Node, kubeVersionSpec string, osVersionSpec string) error { - ctx, cancel := context.WithTimeout(context.Background(), constants.Timeout) +func waitForUpgradeComplete(node corev1.Node, kubeVersionSpec string, osVersionSpec string, + resultChan chan<- error, wg *sync.WaitGroup) { + defer wg.Done() // goroutine 执行完成后减少 WaitGroup 的计数器 + + ctx, cancel := context.WithTimeout(context.Background(), constants.NodeTimeout) defer cancel() done := make(chan struct{}) @@ -215,14 +218,24 @@ func waitForUpgradeComplete(node corev1.Node, kubeVersionSpec string, osVersionS select { case <-done: logrus.Infof("successful upgrade node: %s", node.Name) + resultChan <- nil case <-ctx.Done(): // 上下文超时，跳出循环 if ctx.Err() == context.DeadlineExceeded { logrus.Errorf("failed to upgrade node: %s: %v", node.Name, ctx.Err()) - return ctx.Err() + resultChan <- ctx.Err() } } - return nil + //确保在任务完成后关闭done通道 + close(done) +} + +func conditionMet(node corev1.Node, kubeVersionSpec string, osVersionSpec string) bool { + nodeInfo := node.Status.NodeInfo + if kubeVersionSpec != "" { + return kubeVersionSpec != nodeInfo.KubeProxyVersion && kubeVersionSpec != nodeInfo.KubeletVersion + } + return osVersionSpec != nodeInfo.OSImage } func min(a, b int) int { @@ -231,10 +244,3 @@ func min(a, b int) int { } return b } - -// SetupWithManager sets up the controller with the Manager. -func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&housekeeperiov1alpha1.Update{}). - Complete(r) -} diff --git a/housekeeper/pkg/constants/constants.go b/housekeeper/pkg/constants/constants.go index 2a2a73e..05c167e 100644 --- a/housekeeper/pkg/constants/constants.go +++ b/housekeeper/pkg/constants/constants.go @@ -30,4 +30,9 @@ const ( SockName = "housekeeper-daemon.sock" ) -const Timeout = 3 * time.Minute +const ( + // node upgrade timeout + NodeTimeout = 5 * time.Minute + // time to sleep after processing maxUnavailable nodes + NodeSleepTime = 2 * time.Minute +) -- Gitee From 50ad8d6633f1356d1b251b670954c8ed06c7ee3d Mon Sep 17 00:00:00 2001 From: duyiwei Date: Thu, 27 Jul 2023 17:16:01 +0800 Subject: [PATCH 37/38] Optimize the SIGNED CERTS code --- pkg/cert/certapi.go | 12 ++++ pkg/cert/certs.go | 76 ----------------------- pkg/cert/signedcerts.go | 130 ++++++++++++++++++++++++++++++++++++++++ pkg/cert/tools.go | 16 +++++ 4 files changed, 158 insertions(+), 76 deletions(-) delete mode 100644 pkg/cert/certs.go create mode 100644 pkg/cert/signedcerts.go diff --git a/pkg/cert/certapi.go b/pkg/cert/certapi.go index 21304f1..4c7c642 100644 --- a/pkg/cert/certapi.go +++ b/pkg/cert/certapi.go @@ -23,6 +23,18 @@ import ( "time" ) +type CertInterface interface { + // Cert returns the certificate. + Cert() []byte +} + +// CertKeyInterface contains a private key and the associated cert. +type CertKeyInterface interface { + CertInterface + // Key returns the private key. + Key() []byte +} + type CertificateGenerator interface { GenerateCACertificate() error GenerateSignedCertificate(commonName string) error diff --git a/pkg/cert/certs.go b/pkg/cert/certs.go deleted file mode 100644 index fe1b4b4..0000000 --- a/pkg/cert/certs.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2023 KylinSoft Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "math/big" - "time" - - "github.com/sirupsen/logrus" -) - -// 使用CA证书和私钥生成组件证书和私钥 -func (cm *CertificateManager) GenerateComponentCertificate(componentName string) error { - - // 创建组件的公钥和私钥 - var err error - cm.ComponentKey, err = rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - logrus.Errorf("Failed to generate %s privatekey: %v", componentName, err) - return err - } - componentPublicKey := &cm.ComponentKey.PublicKey - - // 生成一个介于 0 和 2^128 - 1 之间的随机序列号，并将结果存储在 serialNumber 变量中 - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - logrus.Errorf("Failed to generate %s serialNumber: %v", componentName, err) - return err - } - now := time.Now() - - // 组件证书模板 - componentTemplate := &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{Organization: []string{"NKD"}, CommonName: componentName}, - NotBefore: now, - NotAfter: time.Now().AddDate(0, 0, cm.ValidDays), // 有效期为1年 - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - } - - // 使用CA证书和私钥生成组件证书 - componentCertBytes, err := x509.CreateCertificate(rand.Reader, componentTemplate, cm.CACert, componentPublicKey, cm.CAKey) - if err != nil { - logrus.Errorf("Failed to create %s Certificate(caCertBytes): %v", componentName, err) - return err - } - - cm.ComponentCert, err = x509.ParseCertificate(componentCertBytes) - if err != nil { - logrus.Errorf("Failed to parse %s Certificate: %v", componentName, err) - return err - } - - logrus.Infof("Successfully generate %s certificate", componentName) - return nil -} diff --git a/pkg/cert/signedcerts.go b/pkg/cert/signedcerts.go new file mode 100644 index 0000000..66d8a44 --- /dev/null +++ b/pkg/cert/signedcerts.go @@ -0,0 +1,130 @@ +/* +Copyright 2023 KylinSoft Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "math/big" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func SignedCertificate( + cfg *CertConfig, + csr *x509.CertificateRequest, + key *rsa.PrivateKey, + caCert *x509.Certificate, + caKey *rsa.PrivateKey, +) (*x509.Certificate, error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + logrus.Errorf("Failed to generate serialNumber: %v", err) + return nil, err + } + certTemplate := x509.Certificate{ + BasicConstraintsValid: true, + IsCA: cfg.IsCA, + DNSNames: csr.DNSNames, + ExtKeyUsage: cfg.ExtKeyUsages, + IPAddresses: csr.IPAddresses, + KeyUsage: cfg.KeyUsages, + NotAfter: time.Now().Add(cfg.Validity), + NotBefore: caCert.NotBefore, + SerialNumber: serialNumber, + Subject: csr.Subject, + } + certBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, caCert, key.Public(), caKey) + if err != nil { + return nil, errors.Wrap(err, "failed to create x509 certificate") + } + return x509.ParseCertificate(certBytes) +} + +func GenerateSignedCertificate(caKey *rsa.PrivateKey, caCert *x509.Certificate, + cfg *CertConfig) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := PrivateKey() + if err != nil { + return nil, nil, errors.Wrap(err, "failed to generate private key") + } + // create a CSR + csrTmpl := x509.CertificateRequest{Subject: cfg.Subject, DNSNames: cfg.DNSNames, IPAddresses: cfg.IPAddresses} + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &csrTmpl, key) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create certificate request") + } + csr, err := x509.ParseCertificateRequest(csrBytes) + if err != nil { + logrus.Debugf("Failed to parse x509 certificate request: %s", err) + return nil, nil, errors.Wrap(err, "error parsing x509 certificate request") + } + + // create a cert + cert, err := SignedCertificate(cfg, csr, key, caCert, caKey) + if err != nil { + logrus.Debugf("Failed to create a signed certificate: %s", err) + return nil, nil, errors.Wrap(err, "failed to create a signed certificate") + } + return key, cert, nil + +} + +type SignedCertKey struct { + CertKey +} + +func (c *SignedCertKey) Generate( + cfg *CertConfig, + parentCA CertKeyInterface, + filename string, +) error { + var key *rsa.PrivateKey + var crt *x509.Certificate + var err error + + caKey, err := PemToPrivateKey(parentCA.Key()) + if err != nil { + logrus.Debugf("Failed to parse RSA private key: %s", err) + return errors.Wrap(err, "failed to parse rsa private key") + } + + caCert, err := PemToCertificate(parentCA.Cert()) + if err != nil { + logrus.Debugf("Failed to parse x509 certificate: %s", err) + return errors.Wrap(err, "failed to parse x509 certificate") + } + + key, crt, err = GenerateSignedCertificate(caKey, caCert, cfg) + if err != nil { + logrus.Debugf("Failed to generate signed cert/key pair: %s", err) + return errors.Wrap(err, "failed to generate signed cert/key pair") + } + + c.KeyRaw = PrivateKeyToPem(key) + c.CertRaw = CertToPem(crt) + + err = c.SaveCertificateToFile(filename) + if err != nil { + logrus.Errorf("Faile to save %s: %v", filename, err) + } + + return nil +} diff --git a/pkg/cert/tools.go b/pkg/cert/tools.go index 21a8a7c..4d10d39 100644 --- a/pkg/cert/tools.go +++ b/pkg/cert/tools.go @@ -49,6 +49,14 @@ func PrivateKeyToPem(key *rsa.PrivateKey) []byte { return keyinPem } +func PemToPrivateKey(data []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(data) + if block == nil { + return nil, errors.Errorf("could not find a PEM block in the private key") + } + return x509.ParsePKCS1PrivateKey(block.Bytes) +} + // CACertPEM 返回证书的PEM格式字节切片 func CertToPem(cert *x509.Certificate) []byte { certInPem := pem.EncodeToMemory( @@ -60,6 +68,14 @@ func CertToPem(cert *x509.Certificate) []byte { return certInPem } +func PemToCertificate(data []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(data) + if block == nil { + return nil, errors.Errorf("could not find a PEM block in the certificate") + } + return x509.ParseCertificate(block.Bytes) +} + // SaveCertificateToFile 将证书保存到文件 func (c *CertKey) SaveCertificateToFile(filename string) error { err := ioutil.WriteFile(c.SavePath+"/"+filename, c.CertRaw, 0644) -- Gitee From f6f4791524b9aed44b2a80cad94580d59bad7c2c Mon Sep 17 00:00:00 2001 From: lauk Date: Mon, 31 Jul 2023 15:02:26 +0800 Subject: [PATCH 38/38] update readme --- README.md | 35 ++++++++++++++--------------------- docs/images/arch.jpg | Bin 0 -> 181390 bytes 2 files changed, 14 insertions(+), 21 deletions(-) create mode 100644 docs/images/arch.jpg diff --git a/README.md b/README.md index fbc0f5e..ee125ec 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,30 @@ # nestos-kubernetes-deployer #### 介绍 -A Nestos based kubernetes deployment tool +nestos-kubernetes-deployer简称NKD，是基于NestOS部署kubernetes集群运维而准备的解决方案。其目标是在集群外提供对集群基础设施（包括操作系统和kubernetes基础组件）的部署、更新和配置管理等服务，从而简化了集群部署和升级的流程。 + +#### 支持平台 +NKD根据集群需求，连接基础设施提供商动态创建所需的IaaS资源，支持裸金属和虚拟化场景，目前优先实现openstack场景。 #### 软件架构 -软件架构说明 +整体架构如图： +![arch](/docs/images/arch.jpg) +NKD的整体架构由多个组件构成，主要包括NKDS（NestOS-kubernetes-deployer-service）作为主体、部署到集群中的HKO（housekeeper operator）以及集成在NestOS镜像中的installer。此外，还可以配合NestOS镜像构建工具链、配置管理仓库（如git）和私有化部署的容器镜像仓库，共同完成集群运维任务。目前NKDS以命令行工具提供，暂不提供对外http接口和前端配置页面，但主体功能所需的基础设施管理、配置管理、系统镜像管理、证书管理、健康检测等模块已初步形成。HKO主要包括面向集群的HKO组件和集成在NestOS镜像中的HKD（housekeeper daemon）组件。目前installer组件负责在系统点火阶段部署创建K8S集群，未来计划将其功能融合到HKD组件中，使整体方案更加精简，更易于用户根据个性化需求管理所需的K8S基础组件。 -#### 安装教程 +#### 安装部署 +NKD目前提供工具形式，仅支持通过命令行参数或应用配置文件部署kubernetes集群。后续会提供用户友好的前端配置界面，便于轻松生成所需配置，并提供配置变更版本管理功能。在部署NestOS系统时，需要通过ignition点火机制传入系统部署后所需的动态配置。NKD会将用户提供的kubernetes集群部署所需的配置自动合并到ign文件中，使得节点在部署完成操作系统后引导自动开始创建k8s集群，无需手动干预。 -1. xxxx -2. xxxx -3. xxxx +#### 升级维护 +NKD提供了操作系统或k8s基础组件升级维护的功能。用户可以选择是否部署housekeeper自定义资源，用于后续的维护升级。housekeeper的主要更新流程是当操作系统或k8s基础组件需要升级维护时，NKD使用镜像构建工具重新构建新版系统镜像，并在查询到新版镜像后，向集群创建housekeeper CR资源。集群中的housekeeper服务按照配置逐次对集群节点进行升级，完成整个集群的升级工作。 -#### 使用说明 +#### 未来规划 +NKD的最终目标是以长期驻留服务形式提供运维服务，同时支持多个集群的管理。它将提供持久化配置变更记录、证书管理、多种更新升级策略和镜像源频道等功能。未来，我们将持续优化NKD的功能和性能，并引入更多智能化特性，如自动化故障处理和资源优化等。我们的目标是将NKD打造成NestOS生态中的核心组件，为云原生场景下的运维工作提供全方位支持，进一步推动云原生技术的发展和应用。 -1. xxxx -2. xxxx -3. xxxx #### 参与贡献 1. Fork 本仓库 2. 新建 Feat_xxx 分支 3. 提交代码 -4. 新建 Pull Request - - -#### 特技 - -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +4. 新建 Pull Request \ No newline at end of file diff --git a/docs/images/arch.jpg b/docs/images/arch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..24afd47bbbf87077a4279ce3bdf9d33e39dde4e8 GIT binary patch literal 181390 zcmeFZ2V7H6)-W7Iks>Nxs#2w^p$UiuLP=;6dO}$`1QI$3h!v!RgeD-pCJ>Mk0--6= zi?oCe(xod%QLub*_uZ9!cmL1(Jo~oa_kDZv%f08$oH=vmE`>Zu3zv=;xi5?V?)ZF zmh$BRcmtpSO@QjJ_EVlI;GPQrT;Br#sGt8%vrPp62%!MLg^Ay3LfHVoxex%LrvG=^ zZ=HD9c-j1!-AT&zgrg$>uu}v8Fq#7ZECT=lJ^askl*=E;c8QY3Ly^m!@^J#V0UQ9A z0AK(LULQEgc>GnR938 z8O}1$(J?YHo;`Q|JkxplGt4Z^=UFKD`Cm+^ewCy?NlPhso`H^mlKNkSpRWN-v?tG< z+@z)w0Gwc=qGqD{*$UwOWoy(_zu>RUJ$;IX_9QjkiC@`>^8mmpYN`__&oR=RpgBcL z!vLT6!;OnnoRtE%xsBsxI@LmNMI zf{=`?s^??dkodQAoWdd+P;Z}{%W`T@LJKP@HMQ)#!gAY;K$N<8{y`D|rKS^9G^c4P zxynowPobiwJAIPsgLSe@iy;)IWEe(x?MdVSK&v8UwZID1=<^YAKk$)sFdrJkQnFH2ZfLJtGW!}I`G6KfQ#1*AqzIbIqBG$S8y{U=VTmK9`YA10d%}_CEtW-E8 zdFP987r1(D2z|3njI03@ii1*(T)M<@c6_WQqtZU3tXM~602`$8?WgT`9^IC{ zbd?~t>1eL?Xlcpcz*!Wlt$vdp>pc+(Ex2OY&ZWQ#-;z)pq$6t<@t@m@^Pli%)WdEqTYMPPDsG9T``PJ5N4ewxN#dKF9P&fuTO(;0Q zXlxWFzVQ}19b9P=ef6ktwa5?%M&$h{JTXPxWn41kEIZDYwxlPlbP~i}DXbkJS=r4{ zQ-IR|gC81Rds4ZRe68!$rR^CF{1!SaM{8Be>c-Y-tC6386L{25KrhVz_1)5CNljFv zQH+O!vg(Vyv!HuP8q1~vmBm*&@8F(g375tc)P|%zPjW^G$Vh304PlZ!Eg6lgO?E6X z`o@(tT)0GRuVd^B!jP*2&_CINt`;qKDW!0my#NE4cx@}?+A`GSyg4PWJE&DFrY|}) zwPv(j&;g680WBb)Iz2xGY4%;-#+T(*2sxz7l_XA!&_@$@jovB94SPS5`;%`>M6zw9Ve`55kk(rtuiEyagSF>iRG5OvR zSUgviNb%dd)-I%v^3__<2)r;D2-Qi-NfATbmcM#ITjOhG`JuJGq;gAymm#6ADLUVX z^F{ktzu~5##Eby#1cRrgPCR1n7MjkcpHC&|l%cz-3VLe=WUkyh0?(SvT`?rp$x_un z`U#-z0zts>@yYoP+tI;qJSRtz1k~U7nqo)@3lKVAuNVTlw-KKi;_B%L1|#e(3aJIC zGk}g{rqwzIx`7Jb2k+{orsBOJIb`u`!mK+Ej8{__9{DLxaaoWp>RX$16=}^s>ZHV8 zYw#0Z)On1Y=!F;*gt;1?tVG zyQ;BVmSb=40`(OfO}NYbn$~YHwSA*iA(NYc#0CrKxP=92)~rRn94>)hdwHjL8398VGMM5d&t?GB;Bv=r zQi#zn7$1jq@i;hHY}M`Xo7N8g=AehRM@kRIrE5E;%-1AEyRgw5L&|L}>TrJV2mdHs z19yuugeB(*p?#J}{kQpEap6*vwo#gX+7~_x!w726d(3HbN9_6xh0D@PQ(=UF07cF2 zsuvZ!#676oo5W3eJCY*PyfJsGMPzq&Ge1oY*Zt&5`Q&+KgsO2)ad-E_k&S;ADg6Rx0ftnE(~hl@9v5yWxv1RA zdDQc0n6K};mmJAND7wlK@0Fha#vsKiGPQk~i1MImVhOU2@;9h^^sfX3{~OB8rOET+ zKhF#)yV<5U(%f}*8^aO!>Bya4ELy%go$glvG=vjDD<92akAd{FN!Sfx%JdIaRYF>=WMY67uun!^o zo^QQ(ShEtkK#f<+XU zw$}xRq08`TD0`^$2ZzpFLBVOg$K3tC5@kEza}01PSCvzrrP?m+8jH!lC7WHH??@0Z z5S~CM$bp8j0x; zb)T*)^4fA@DrjAiyqktXX%Al1Q(4PzzBg&!`4LbHWJ45augVxqL>j?#u0O*qd3e0= z1S>@U1ROcwv;`IgFYzRyYy+2jukSR*>^~Z7XSb>z70;7?0IErkxQV-|yoDQnU}PEr zgno9hK5VVa1)m~1y1c)!?DF`Tba6eW(RQMSg#7)UZpNE0zTcroA0;lz=IS^l|0olE zo(t7wkQwOg(Ys?4Rhc;;ztnvBqS>U=Xtp^jp;t4?7;kH|R$-@vkduv!oPzXrCgws$ zLd9;Rx#YO4H~R5s_qJYd=Cn=!kz@a^RnAdDe%fVBij*=)`(AqtYvRRrT@uVTx%K*-NmY;p ztPu!3BdFp_I@x0jXW&`Vjo*pMi-#y(>pO^FZxp_4pJ>l>>iQc^LyD#nQ_o=`_uPg7llUVv~ht`jq#%S_=0@)2lROF(`y-1jG7;!NQPL?BLBDSo}%$QS)4D}~w6wtj0s^d4!4)72f25NHf<~{{C?Sm> z7bZgb)2E?qN)K8wgWmZ971Q?YNZyWT5yqrxg;xVdUDD4jX}g|ld2RA+#5c`2k_d6} zHPdu$f1r)=@nE*its1BUb+-h>m0But_^{3>(l1f242@J5=&)p(B%05X%A{mRp8ywc&YQ9V1*Fw zF?#BKw^l6q=uknd=tbm?(E&}u+<7<_D4V(Jn%%U>8z;$K!fEEmm<7X*C-Ioo@Kw3) zE~ueb%VnS3N$YYT?pO#2`ic)EJ77{zrB^LmONN4)z2%4T^@HrA^h5`)1a-vBqoS7@ zb9GW@l<<&2m3$27$*lt)8A00ii*0_=s^7CG^~fKi5b0K^_qxS*bB%Iyb^K=HwWU6E z710j2pv|WtyCn2{XPi1;sam9)Mby!qnwKT-TlMELhbTVw6JXgQ@oY}vps_{4 zXSj0nnf&WUeeW8JzWc7LVaDor`wKFt-}n#;22DB+x0oyx=$ae1nCc5NQru4Sv}3@p zeZMz}-9rPNui}Wt^f-+${X4RK@)q6j?%@6N8O-D8=#)+v2Ti6f!=2cQYsPI}8yQ+1 zKK5H8r9GR|MZS}(>C*P$i}Q$(_1J_}^V&*Zp#rA-YE5T?rtM_VH3OGaQ-wta zbnSFtLb@$9TQ5fuFagrq=2v8D@0<7NoA;IGJPJYW*GrFOLTeQJOmG>bXI+zg3;A}> zCM`s(m*972O4T0Wv=(Fx<@OJ^+G@(riN6fnec2kbDjQ?26bymD+5vd zd|L5m%ix;3)j4MyLZzh9&%x&V5J4!(si%0)@u1}3?E6oWFM@6sNepZ+Dys-D>y>>-c zJSryTdvfm{4tX)E=M7nkp7hUH;wHVo>PGw2bV>EZF?Usr!?JVAxMU&)4PIPGfA+mHa+UgqVTL)p;&L z(wi2WHlZ1rpMcF|ykyk#=x1>p4l3Emo(Fn2D{A`7Z&yxVei6rBf@sB%ZQFfF2p2ZC z+N9KYF;EhY9vUD!tU92w+n2YUDgL&0d_z5QXG^nkofRpHmJ&Z1IWgC6=ShcvO?kXT zI$*NVc{1OJdJ=rJy*JKg)BQmfZ?@zL9oc-x67~ncH7+PKBv20xnL?oOI zobYE|sONC)ODR(pnJAbXS~~!Z4KEA}3P>1&lg}fS7Lf76+U_ARaG<;M^0PoDCg*(| zbgyy0zMl{>op@NIO(-$g3`8uCU<^{TUpwGY)jcdDFz%@`t^tdp#-4Er_~a4idj$<> zhk1%Pl1*u|&hxvmA*UdL^C_JH>r-J(TV?5ZAYQUdJHT-*sdmB#Eaaxp)T40C0sGGF zbzPyOY@KxN;DSY|baE;*NX4{etq#_EmSi_cCH-XjCfp%ah=5 z%5680e&#Z3B<=g_iC;JFU_{c6yuO?S`GSTlH%uIeiOFkFix7}Zt%Rcmujh|`h<#LS zYc?5?bi~x1*uB_^Y*ab-(paoO1QI@_2>>G*<3%IrT!hrKi!5KD8l&W0-tD>eOHKux zSSBLHdx=RJ`3`(Cj?;3B@>5#6S=4Nm`fl-<;@ugUK_O@~Rh7JH)4tlPO}M^GcUy^_ zCJE;NJLa#PKc1F40A{DHRv5PCI;#ou8DS zf#?i3CG8|Pq@~5&p{{v_ud<#vjcAPDT=87>6DGSsOU@Wxa7#EFXA&H;@>z<*c09d4 z4i(SzVDqGUu2vK^v^T}kX)N;X_%zfi3jI;Jpj6#M&~o)W(Q3}8T`j9aD@MNP)|aQe zPR~AtR#fE&bqK!am$-E}S3!5q13Wv16xECAu&FdzW5TR97804Y>V5(o zE!^!Ezt;lWhm+5Q_E$Lf^D2djG0Z%bo=xCPG!{_pogui-Oa19=H3e&4oRGSCO?T%bUTe&48pg}@ux*GQj!Lo;*B4?U68J2Sr zfK7qr_FUhHfi4CwKtfQl^Lfk%{a(}(P}bFwp8&ik#N=gYHtofY)iysVh`eE!Zx15f zsMc-?@{P-9HU9bM>XnU#6z1?XbIwY~g?BiO7)A}3QOlvB@#vaYt|Hb~CG9sBGH`Yo zZy$aLO6#_ux8vc~_CBa&d@JORLKP(>LBRLz6sw+{S`Q{jmpqIKzPTVbZAh#|JzlX2 zTWP!Wg>i-{=qNq&BXrc-OZC-)i!WZbk}M$zP9}O?IQMRiU6|XgEl%Ji zH|n%{q2RTPF0Jx9pJLYFrD&CLm$C+;vkiBxsF7sX5ty&i%a@DS@A01& zK*lSwfl*=)E^2h8P4K;ND~+Ky=Ld~*#;+0+q>bXEx7=E5%6czk-Ef?`V2Dd=3nY7X zxX6-g`hb#!X|e*|OwXUpW`RXX2*H6~Ja9D8J+;a)K#}{>fQ+6(m-qIFCLy~XjKFhc zlJ7($eqePqGB>aUY8MNqC1!SsBp76gp>=Oc+EhB+oGG?8McFqMIxy4=8q$YaV)P!7 zXupdKaVgQ?Fwq(6afGOBb@pk10^`>lc~Wg{OCp!XmEvPpv#OGaEz;cPDUx5x> zi!P?FRx;fv@8zK?Py016L=fy8JBW&%8;_(h%%A1vic6cT=h`nEN)I(v8=6R*s~yBt zOQEXqIc;LqIU2bJYY3G9UuJrf4n-`viuin7*>1`w+GEl+l>KE1f}0%f!G_^+L^xJL zY0-{}&~dSHU@bTZCF;?rzg2fA~|6) zvw_+F*_A6fr%H$H*06c3uDWaV=lCgk3=8j_>k}~y0#KN2W+XT{-_hdfm$e+{hw|mi ziZiWh_bT4UUX8PKYUF|ciBTULn)7X<`NfBz^S?>1$SZHRGx}!kdLkdm-NdqoeMbDS z%h*u+P1`W`vEzc=NYG$g4S;l547=D@@-^1K;_TmORX^J`^ZaSl{G%rO7`5SV)=Gb) zW$26j=K1vBV*EvO|3z(ZRN|9(lRD$Yi-&cEM(|}3ieeBgyo`*zkAT3J_9^Auktp4W zh+SV@pV2DC*8pDPf7Y7*6}r>OMLLBetCcI{wEHf*UpExlEx7y&v4Z``)3+(?Sq5~wi%7N#A~LIj!ps(wUc4ADpmHWIG5nzs_k*X1Awl?l8*`Rs zL5t>Tt~0cqal9exWoz7bn4z%L9j5zsm;RNe{iV!_J30r(S27Vsvk?Oy|^i?aVi=4aI3f7By<_Q|Tei>Jl?#_1}p>rC^5N*C*mAE>+raG3rf z7pgObhCq4U&hb0XWEz)T3~>@^QfD*ta$e`)&<;jQ_GBqUy8iREmkNcAJuS@s;FDmf zGw+s#&D@F7emjrPkzur_4H}{r?2_(JB0gXB*mt&K+x;8wiGLfMe-GpuuTJm= z*Ia9PaMw!eMy;>gcO$8D%j9nz%41>13~FET53jx37c>KmPTKKYUTlB`c&v^8vv@$` z6;*jn`*r=Ud(}I0ZRG{97%@Bg22I<(RdA z-a7ZjxaxF}l3#dX8q#+-X%| zrPYHZI9z8rm10}_z$oy`>8$?lrQfD`+|+s2|7?Og|Mr;we|98%Yf{@N3%0XKhW6Sk zWv9I8>R7)r?IFb>M)s&2p+K&|PR(*LM$teiy_Ks?j#>MkIx5^oQR0wL&gQ1A&Y;9@FA3__}bOG>HLNAp8$WW zqukS~4ExtuRKJIS25El+V1*(dW3_h+e7qm|Vr@}h7Mmfan%5ZsbUt<`C$2rYHG+w# zUqF8sv*elOMZ8FFQoJ&hH2ChsYpT2pT>sAg=W2PHx}ohSm6Wt2m!ef-0mQH_QOl+o&Gpw$F=B?3+p+p}Lp;D0&(B zbM)d2$#VVBxEXk*mwvz~LO^Pf8_Th~e#wSToJ$RWOico6 zBO$#kN>#*)xJj>GkC@yO5gf>cn!W|6OP5Xd_!`=18SI}o6GH0>!9;guuGRR~P4meH zn68*~$i!kQu*LmnF|R1sGqqO z`+3#X0z5zYHSbuf7kd7en@&3+6l;8#lzNRr>aBFWQi_iSNwv7gQCYv!mJ#Kq!8&hOGYS93 zz!;y#4d#r`3@u)+7t59DiQGU`MeoeK))$;4_<{RI)pJ;`So(zuOyiM<2}~;nViI%t zeJQ5MB_ifkzUUFHBr{k9EyWth`C{^zpLJA^QT#Nldd%3!>DeE8V+SEH|8VK7}GTq`&J9W8REv8<)F6#eV){d&0GB8a&;9K3?pJEk?x-)yyY z1dStz_vQ&1(_Zt;FuS2Obd4)kyS^j|mC{OBCAa5Suszk=hGg7G7g@Lz1vOx{7=$tF z2Ci9IxL3=Kd)XhpJKOmJQ=;_kDJp4DFPa?8Chn_2)i|(R-K{1HcT6GN&BtVDB{hI@ zH$z{a2C`3V8^M2d)g_(2nJeDvJt1VP%O02o+cAs(H|kT;ZFsn_jU zI69WTF3ij=hX4icM|XjE*{NSh$0Y0+SE00PJ+{c4k3I$lF(jNUgNI;A-A`Agz^Uh_ z-Jd{sfYankz2*IU&B8FYUbtpw?-v$Fi>xaGCY~7r-y6n@rV~U*Sx82&hMm#9)mVs0 zf|}5038=24JqRqbBz5~i$fE_=DO{y5q*1r%ZnHD*sxbzV+;eqSE|Y`b>piqH-9d`= zsRAPc2)-2F2U{%?C4GKp-F2Y~ReRv=kMpSLE7v5X6Gffw`WM8gV!Bcoy-Do{G}WhLU^Rkk9=@C-XM!p^Yr!dXUAL($b?ku( zbFmhbgZRZ&e9n$j(bdbx)^niO?`55x4?iey9E@RFPc~YQA*$TW+(`bSVaHDfuD!oE zDfL;X1`ILESYpZy#f$MRuIV1a;=@w@bjRA6uaZ>M^bVUEn#1>`|ITnMF0vWVkP{-m=d@Fu{N0e2dLq zmIk>%X%5`+KX~hmU#kUvI~(wVi>5m^$4G z<|K5VbT<*TCCNIEKTD#_k^lfDzrQc^>uH%c^(JF&))8Om0ga~qS)1{M*~y5XfQl7s z4|~&YY@5^o5=I_%%Tj9L4id+6I)NDTiTkG$AZm-Xc)+CDdX~ZZ{2AqeFRgbZX^NCe z@|H3OmxcV9NiSy6{i>+=!SI)@p^#$r$Rz12w9iP1!Y%KCNUnG>IOgMXs@L5e z)txT8Kc{49J?%WDEzC3Al7JmR>!$f1#QEz6(6|BA_my9*j~zJ$K^fMsLJ@2eh!&y?*CtWmUu+@tXbkG?TkD#12~l zdD$f=R`}A|Hf0Dbtk9U_+BgXm%2I!H)~-$-*Lk2`FFoDLE#~ zEDQ=o6TQ=;6P`Is1vyz9WQeM`A=?s7cq|S8i34O2hyYisC(n0m6%s50((F^y`z)_PR_6ocfLJVSKS8&pf z;XOUDK08!p=;%5H@rBm;NN-d64yt~SuoA=L8Shd9osFmNJk8$v=4K zdd}<5?_3MI{IK7G9s#7w{z+f-m!uQM&yI3GxQm|uZk!!ioMh9qQa9Os@|LmlCGl(k zK%ge9QR#BuU{jX>=@_1Dd?2=)<#3|}k=0)%4LHH}r+?1hR^IzyF#UeE#-8mnDOHK8 ztfuJ6r4tGztk)1+vTko7{kAFq0I#y%epqcWd+fQ!GFNP#_v?H^=OH281UIngo?{v$ zNLby6gjOr~vHFqYn?}0vM9Rf6-*v$Yyfk0RO;MA^Y-=pnauuX?_Ty_aPqlI92cLH_ zs4bAtplsrTcvLU_*kAf`X{mqHy!l6VBCIbb$aXeK>ZD_|pcq)@ir0NfD5-hT(fTtL zRDMz^)bJjb;5xjEAS>rM`g)>MAeWN3D7$*XjU>F_uV@Qx{z3*=hEJiaCN) zfBl-wGPsuhMhfGBV7(k9=&7LgvqaGzqst*+l$7|@jVWAwu~`(?RGreYw_m#3@WtYW zjCo<@H`#qsm#q(yaiq&%Fd)Se+ujcu>nR!5ESt*4aUv_FlZm}g-7x3PBkIA)PqYX6 z82N<|(fez1COiEBu*?wH{hB4z>O0e#TVo$C>K0W2=X_0*hBo$1zG>~=V-R#;hN+?p zs=wu``asY@eH>!0#ltnvYy4icjeCS#RiyU$Ykh!f8^HE0!(RiBl-L zGe&bl@Y;k%pVoxmWRsA^^&R7)ATy41`9&;=%VAJYM~r!%UnC)(vpvHTza=@0VeWDn zH3T~cx^PQv(zi*Ra$9v@R2&t4u+sf1zb(orU?{QR7BAtVr`JqCi^ra-kD!2g>WXHL zS4^7xaBaT~xx}wV=&{JwWBQR|9gJnXpqT8TZ68hYJoh`H7WpSdg)UtwO#9}(3*d`1Gd}_RrAN#pJdpIr-&j)yR^2eD z_XeG76j=u)2u*GF%ucQdfoygwt8tE%=86rKK!H0&swzg7Iai_xA~(`CP+vl$cS545 z#(C(knBYiQbm12(hFm(KkLn)E-(Fs(Dr&#{$}1FuZg!SBF;QTXw)e0Th#1v`c`-FmLk7)7sfTc-FMyTfi+ zkt#G~MZk>^z>NBcZuZS#%GATvAmdoqla$@L>`Q7!YZf~rjEK&xB8$*5FU3?XSuuD1 z(V@Wt5WaFaLW=C{l%3HSI4oH{!NH8ucy2<9`p7c0LV9v9v2f8?4_jp+|588})0)wCq^k2Hr;MoTP*tgyHZNr z4!M+F(bdB47yKxl0eXFX%nqzoE+IFb5g1*(LxMt8xxqq5 za?Xt{g1{0OezYveE6YQRfUUlK`Bh4iF1BXUTOjo(U>{Y0$DPdma$uDEBQI7w*%#yN zs1hL+q&Hj&CtVNqRiTfwF!{DZXUQ|8t-pp}nP)A7&ZrDa$u07FqRZFLN_32}Yt_xn zuKJPX)%8rCfeQHZxd<%fNC;3Uze&b`qn-Nse$hmVocULD2?^zjGW(*MrH!-i41dzI zMM_YDI}A9(1`sZ~Z!ce*L4t1@ySwJNm`!>EE$>a*b7x(h>%r8rnWX1+=96L3z4;zM zEE8O=;2b})II*W&fP8riFLfro^_EjK9X{0k+?45!BLe4}8*7UtX$wWBxYR0#1Ya70 zAt#C;l`ACt6fE4>GWFiTZ;!b*#;SIuxym2tHu$_SVbqTqH;5!mOF_Lb+9hxO(#3@C z=Z980CQ=Uc54v_!7ghG<;A8pP3G$}-$OlWUOIeU4xL3T#xKDD9K0*fJ>xLP_Rj)hd zXLlwh`SsRYRvwc5BAd1tj_fghy9IYw%p(~gMIKON-W1)|B;GnVe2WxKwFgSFKcEU+ zvOub%X}e-GWo!xIxAq?~YjhEsxA2Pdvffi1xG0+{c0)qk2VE&BpmOf{?vfza$e^dd z>xmEcFRt3jamgZLNIjC(PgJ>8UFd`g#H3XPC_uSVkskp7|BI%b+aos0(0sij&F95i z-O=l`zL#@q3WPcntv|_-{Nnrf7>S-B5vM6P80U%L2uLRJ{^SnZ>CkN8Jn!sFbLAF~ zcd}t}kPKvj0){X0TFoM3;YN@6$ER_k>}{hDr)4k^IG@77CnSR1FbPKj2Xt7vBoE-d zE}P1kw!%1F`_dIQrQ$xF?XAAS@?2`7st(#OCVScQh?VfNm@MpL_-wpgZ!Tvf12WXa z8t1JhGn$a%5_SJ>jnCw1!S~~@-whLhp%tilwUQpyloOKSXKH)XycRFT(E111Q~oAI zm7o}j;#{5DpYD^*Gy67g0w%97Pl zLv0cMr?F@f&ah;Rup~0xxONRA%TVbnR=6tTpmk-}e&>c4w|4W^^#w82@XjuHQSsu` zOXw9h?V9!hRx_3hmE~jA7fcpYuccfp1Q)hzGmFTF(Iz6L+Tw)+opbN}1f11=FdmbG z%h;AdUV5JVv0n5!Sc*;v$~_QHL;t>ObVhaZ-A96JqFb{*!42W75*ZB4g2phX@apB> zsZ+}B@oFC83i61_)MQmJ+~EdFDMb6@MT9GBJlj){QX3h53PBBfH%AM}baHTP*^ZU} z{g*dc2l6!LqeqJ9#O_>*0R^7_vT!R*deC~JCTn`>?4MeU&Az;6-_Xz4T7Hjcs~)dk zP3lgw!;N6{NY^#;Zf?zS7F59zcS^sql#A^=+KggxW#IbS!{iaidPVucwqLwJm3W3l zA)fQDnG3Q=QzygV%EM4_4$>ngRoI~k0?seNA8v|_<`|(6yCZYMVoZD)@@iqZ<~I3U zQ|3Pbst}!@fW_;}(_J-`F+b0*vBy?;oMhtZ0O6Kv3WK{D8KrJYS)MJ3=f)|tFnL6f zY4B1)jxKj6W3x$OcE?kA!3Er;&#Q%*WA;_Nkx z8P)6?i)4b|r-K0$$DEyF4ar4s77=jK(rnWC&!h!VxS!GNF%> z^fk}3A~^@oiF?*_E0XXX7GwtH*A;~iiFtlmI6WRT}FL7SuC6s`RTF41;YT9Du z3p?_YPoSVB|h#*9LX(}ca^)3l;Hd+`IkAq#e~>qmNDwN0nFs7SN_nXc&n)G}4gVZ8U` zIN#f5XR{v!zt$XHgStWIolA23mGQiyNP9#34%Mi8ahB|k)~}Jgo++A<%f^SuivAOO zF-C2ig{Gyhg`#~xJRowf-I7lt)pWI{iUP9MJ+u5_3tZIAm z+PuZT%a46{v>r{oVG1lCM#kP_RPvNdv}3qSTYjvR?{8)_8Q0UguZ0$$G%W+JY+H0U z>((gFDlJIIvGtbhPTFNs>%G)Vd~9Jtm9nYU_wWwA5{Z#^2hZIwlL3&%EpZyaJxF zpKGBOqbw(`So@ewX*{;msw?A(dti;9dI+!1H9ea=bpao7p7;5jW^DfTDp9CCzi@*? zVTs_aS%q!Lh^=dCD7Se;nd5~&Sgq!@|MzP3zrLjWUsuk5lwjw-!E&$QN2|G=$M*yKp}Zo; zBcKpYjqWDH3O+1g7nU(#bHDG_=3LzM-+1JMYwH(ljq*?`My~)|$g6+lce^23L`_rF6ohgO!Z&x4IsP??xCbUz3veHRhaWHgvG`%;f)dgy(nO)m0PH zPk`{3k|ANu7ag#)Prk*~Z~g<4w}w_Uxx+M+Q$EKFzAbp<-L78<;C0IaBcCU#6$hcOk=$M zhw&5u75%GY273$@xtDhZmVD|cb%i{kyDqLa+G;Ll$=r!EF%#m&tAm*{pRz5Z8}9~Y zle7h7Q{$P@PLTEF4y&^c)}RAZg)=7a9vsTJC(_Q8-fb&(9;{-Npd7P-5SJ=OpQf4) zs*wBMj7r5e@N_ptY?Yzd?(!=CaUPZUF~+AYYLq^CiyvMl*3}x{K3P%*=e?(x6wESh zF~#dOw*IEKWF>QLdCnNXTvq4iJG*SWtMBXW;oDhbdFB9zgT(VN_ngd0p{A7Xw5BP(@A<$eT!c~nI2QH((j+E1q%?i{H@^GV?W8C zk01S3@YBB&=*0idszTJ*?GKT4cS>7I#{x{0hdwf3i@O6;rlMw|fb~j7PI5Xd0`G11 zQ*!pz8*{@GHPpu*T`vnSlhKY`iPGwmHtZM=3{~qJ#lJVI1eUNyHkKfVZ93QBUL~Ok z?3NB4gTxm!`CGa6cBIbmFS@k7Lpbs8zf~}2Ohvwy$C!Y;Nv~;bDb#j^HCSyN+YezX#ZI8gHX*JNbTxi18ta(qXs~tV7FjpQX~HlbXVC%#?g+JNjEO0rrcy zGk*x%45UX}_%9E6v9GFKIhgaJ?!Fon%Pu=~Ci)|y<)=A)w@X+1+s*!6;ph?g7ud{< z`TF@bPmXFfv!YK0O^r_`uSQLYx&IN8dg8SY_%j&o#$REy>HoDb+Ebvz`>uiCp#YU0 zC@u|cvu5~|PIbKX@4ooIoicqGOvf*_amgF=t^sW$HnU=kD>0B9TFiTUFh+OEr=&d} zJEQ{3OK!xAX+iJv#QH|AAFZKSqufurTRZL;s8BMbL%XA~>}J za+&WNJE~0;JPLW+lM-^1c(K&L`l(DJ>drBv@er?#i(kYG^hQQ=d9(7gSiQ2JNII-; zN>&72V=I!dg96dvwo$$-A*X7%gxtmM!=yp)Hn4JC1Q237&QZc9B|NzX^m^)A)v}W1 zc%gNi0x>#RVm{B-N~F^#B@4oY14BmgMYVND45YXbSAUu|9*9#efAL$b3E&EAQ5~++ z3L=$eq%8Y2KVYA&s@M~3kgaFyKPEXT-rml?KT9kcgtq{%h_6)-p_+KfX9DXDHdvDp zXWmQ%ND3|79g>*u{WeuE)@GwDGARqk6*{?0LPmNBU~RpHtMAp`$s_%gzBfD<;{8wn zOEO3rcHt4~NJtS*8cOT62b!aiZ;}jh5Aqe{>_A_wLyc>i(QJ9`*_@h^t7;v24T)({ z5<_c#WEBgc0O6 zC&qk9kL5`xY&|wqm+nj#7_Vm+&gQg4v;{P>7vy?tf$C8;=Zw4s2cEjpQgLLDo4BJZG^L<|cg~+4!6EC<<Wcb~6)8Z10kRa$OH6phK?y_3gJnpyQyVvHc7sGia`6hb zZ3Q0&1&ljXC_}94MD2|M={ckVNx6JU-Q_)am{PuJ1&-6Ffb+wP_G4 z(XZY$xWg~d-3syNG*S>17302=Brq)bAv@Zgb!M?(L5=&s^Mfi7rN0B2bcD^S^M4kD zc8>^TIt?j8aiFjw*pZ379KC0kW}@emKYLBt9go@oFPoa;f{)&G&$2guPbeAksQ^5E zGcCR=YfK2qpV_meV7oXxzljZ%#J$dbHmyTaH=Z$M%E5tUtdw;GV(y#5dMD3yCakFJ z^&*k5Z320M__j`=FCARs&6ea2q70t3tt~B#(NCmuq7=e@XZWtAuB{!|Wi`5JGQs$= zaYIsiXXWwTzB@(KeLI`*W^>F5ole{6wvM6dkwRBJZ?MN^ekE}Q@!$tXh3{*bt|9TZ z&N{h~x_4oSZcLm_oU%J_J*FIuQQF@2jo%ZtZm(kL3-V=zRG~&m{mG{BM>6`Hx&m!I z%#4}bQyU8q17Gwxyu|coM9=k#xqJ^jraBk%dZ$MXF-g6K@sddAovgCUH!o>d;Z9fD z$xN~`SA8Y?b5;b|K+GGc8XtZZ9)|%|WP5chsX^UV`u1%lsHYW@n}rp9nTCdq6-Jq7 z0v7otO-VEI;4a@VVvhB0s^ zw@GB#+^&$4Bh!@cPiJJ>x;UxV`u5A$Oezz{8}lyR(YGn|f^Rt^+AXv5s{K0FAlcS% zmZ7FKewoKn71Ob^fXG}~CM;X+q^+1JGLB=fw-UvB&c2$RrN`x7Y0Q0E7v^QUy22Pd zY7=OnB6d-BGT+q$lpopnbM{IuaU^u$t5HP0*SdL@f(L-FlxIS?zgi{u#aeNExI3-vyg5z+CmP354HH5+@f$fwTVgqa!mClPP;)|@ z21Bd;I;l9e%FTAPfHI1NmN?WexK)QDP??z^ER%A2t&uV(4a!LpY2(Ka(Nz)oc|+Yj z^m5AW$Q_V&K|oir%_uj}6liL;i!&`85lu`7Ea<<_f~Zz1LTs@TPPu!rvFNug^A$x% zn|0|lS}M(iF$1#$#6-ZiEH86}Wadv`!4Y&z&Hw#yG|m z7ZHplh_5WAMi6ah_fPPgQl}0LGj55R_*E&6iZ4?no66*f$DTgtg+!+MtV{gP5b0?x zbpodQ1!3nYkk=oz!b_B`IaBtYm=i5$AZfrS5{Iv{AtNi%`zJizyq8D1UiYPnv<{vs zpJa1G#DC$0I^X|vZZPt5^^p2FvohAP==Ry{cZPLTl|_#plJ0FwG$krur{_VnAhulu zqvQ%%7)YKF4pW-0xzt!Ac6{=Rw&hZy#KQOHYj$!Z)AAYQGEg5YI6Te+trfG#^7S~` zvLU%&>YK>>RGKF4asM<%$u`kxemMnh6+QsV|1DLGSc(;Ov%LuJ)@Bpn>`JCG5x{Ff zVlx9psgG^j&u-Y>zF6()kK+-rjCu~8(2tj83Si@;DHTtIR4!q}*8N&3GJI9(UPzMv_{NH?!Xu(c z3)LiNSe}hl_F5@FM%=Z;SZ_Ra?!1h~?+nkj0b1DL`$5+{U1iHcZK+z|^b$azTfG|^ zBC4e=Yj}2OU6AkUoXORzOoo+PyS^QDGUkI@(+eLBhn-FB&8oCj=4_SgQwUWWv_3qp z+(vi~JE-kO{!)TCQ_)l=bHZZlwWk+1q^!oO66hjv!Xc$6+erZ_o=Z|XFrx%DiaBqD zca<aP3RY9EU@}7kAXRN_r15!OC&!x6tZnpYcEY+5mUaAx~$N4S9 zAfkTAMmU*a@~_A9``BPXDOq>2Yy{3!0rZn)bS;QoRRi8Ap&YrOaYx#95^C+3>-Eh+ zY4}-d0ojvtW~3LvoSBKkCoGu~Wx^F8Qwn0xetoANl*}0E38eVvpbXw_no|gqYtXI) zLw`(zWJ=2rw$FtySrL=dy%J@r!9HlZ{8Zr`{qM8kXv=8$E6d(J6Ae56yOi4~TMoha zt9N#^kF=|S{q+{KhP5q7PJPQ(odKP!^Gi))rZox4K5|vso?CJG0^VlIw(ka=XJVv(*5`5MiU`g*%W!FN z6kYs3ou8i_TjA}r?&)Fbyl~}>)EvSIogoVq9Q#4>Y%ogqhK+=6*GEcdc!^$_V(Hd@ z+zAuYh7Qjj$QQdo)60J_?!@VZ5BKy#YwMIxC?p#x}}t7{VC=h_spW2mJ-rDl_I*iUx55tZ$2SKCsF_n*PUu7375q{hmz=))^dE^jnLxWd~+NL+U@5OjN_7dPWacu=f=Yy_TF>w5> zZ#11*OI@m7`ta#c8zD4EZYQ7T!SVa0WnBIZfN!$(>WvyJTdG0jH}h!unXM3crT>eo zG5ar+%6+d?&+lh*k!3225;q{~JH~xG$_8O2vRVgK1^4MYq35N5lVw12U4g51HzYRw zjCYWCZr{a@&Ux;mnhk0URU-%vXprn&L+n~i%D1X*xF>E<6_7Udq40JfVBwCavr5WB zDr0hzr&lhsTt9Gn*m5Z_ZDbiK_dfb!(U7UiD>M-$v8e2;ufCni?gQ4ksT~0$_d;y8 zGct8~ei+98lOK*d=jYo=n%T9Nm8>%7c04av5~ta1-OY;CFXl-&$zB8vEWRihJs7!= zVCwcN1DZEA5NjtN@g^;j-IAKz7=k_rEI~7QhCu9e;ku6N`BN&s0JVAIM6eNnGk`TI zA>#Bu>6&*xVXG=8Mb>|h{PzLmtIj_Ns;c{rDtMkmY~Jm*=UPb zkZn9L*XX&RC{0xQ#@WMdDn+2Vf;opuhRhx7m0thj{|=i}+fXc0+o46RL3`3;g5P)4 zv)KUSc$*eWG4E8;4x>ohsI?4~YmM_JiCRTK6ts*wJXfBr_0A2Z-kKh3*C_FP{Wz^` zSo~RXClWl2z0#C7twbDh$|Vkg=sAaQoBw1rU9!@ca63L%l$hDl`#g?1b-gE&6RS&6 zA?~PfBcy9rQa(2xHK{w5IV~no)Sx0MH-_g2OY0`2VR^{Z~)R-!q%Mrm=T3np_W4!j33eR2>Rkew3Z&d(P@25O!0=?TF%ow^7SKqIdrR zm5MkcH0JHU-}Vl^`lu7xv1ivFM6(gD04>EtzroyAvdKQPqQS#;CJZdc29y|ok&e%h!AZGIu~t;|-4^U_=QOA80gKX1Lf8>97-IIJ+5)*g_-y)Z)Qpj_(o zReB_=il1lI-BB2}xgC@>?IPV?_G{NOHF9^9?)#7xpe z!nw8prBh%RiK|H6cvs4QQb2AgB3N!*Uvh4S`BQp_kAPm$Bl&`4FpSJI4`ech; zGL&^%SaKo>EdjOXxB>Zfqear6q8eN*LvlpGE?oWQrda#^zv?K za|+5ychKI)1@pU-3iom1`QwDXhyezrRj)Ji4WLZFA=%ZST+y3w^k~mP2=75QzlsAV zPHv?tz2*f$yauU6tXwmSD7!Xc?((gQrMH?|yQ0~1Lo*zBqDDY}!c6p-f`CM}T=%dl zPc31iq<>Ka47f0qH^Y-~M7<+Ob9U{jc8T>f53Wd9sY(QfhabxfXO6FY;yXdd_CLn6 z&gn=cO)e~md;;}fS5~{nd)#06RgweD=o@>I^hY8$);roeB{B<<1L=0N!FLC>H=;lW zUD8jFdc^MCZoT`cLSgaw#m{kwC+G66T4r0MLNf{%g4()`QdUa=O-AuPK=Bp*K;JWO zeM*(eeNJSuo}aImgt&6=i(e6t)NlMb_awlpK4??cP3xl&qRukVE!S0FANd=@1Nc4e z{hFi$!!5xuk5Cx(yY!}))4}5r`0sXi7+er}q zM^BU3w1jb$_52jW5v=#0dT6;u+rPlJGwrAWPn69Yzw=#EmS)DoY zKzb2r`7W zVF4SF$=LlABL@wWu6_R$6vifzRLMDIh0~3#_5srKseQ9*w(BXs8sLG9NrF_Kx%a_% zfuyKGz1fz6R$G(#3Jz$a7!eTY%8BoQAh@rYP`Lt+pOVq8ApYG$c*=-A<)X;PG`$#O z6iHei4~^GV=3ge0L!LLVV_QvzkWtG3CD!_;VgTv%RNY&>WVZF zk)ODu#R?rxu-@u0AT893XS8c1!AVz+=8C4OoUkCMN?c5V=M9wWj)g8L6~+NAT=Uk2 zi8|B|W=pk_J;&d`-CC#so1}U<=Iw`yqliz`2XwA7th_YqYi0YpBw#hjb;S*Hy~dp; zm&AFZi6!rcu8H_lqA`AN%%K3 zIor1<3j^y`u7uo^n2Tea%pAXQs%9up@dCeAx|iFdUFs-#SS7nkQ8s7`is*j7I8Gx~kzGd5O+sy*Oh) zx86#!f?j`RL6(?~2;fyp*uB++JJ;#!GueH3YNc%P=)o?v5a@;w(9QBe>z)CTqz!X9 z_}u&2Dx7mZ!tYD*y(@w1{6M*!>#ou8j#I`g>t6Q^Sr<9kMaK_)vTRZKo#CDh$yS7y zoYv)U*RI*~;Qm=_rS3kS)#2R2uE^i=dLp9FB@5JIVhz(~xn%W&aUMeoK6!F5v@5hy zSpa5r=B}#d8ZxC^Ke}3_NfNgD8I4E#E)i5E4Q>4>#R_?&CH`!3xosQi983Cvt|*rm zHvB1LFG~4UvLL4YMnxTAaQ3j$Yb038LdefkQ|>c5oLwm>!`z&V1UBD)hH^U`Lt)2p zu)b(7S71}7Dj&F9+du?eBEx)uJ0F}CB5^>l8bKpjJOzyCA{dlLTLe2RvsasD2H)5! z7*!MQvZ<(!mhMk?emyBwpyFcA@Im-crN)Gc$5v3s6U+BOB7nKe;DA^m&0}eSzph?b zMNQPSZwiFK9#YWSskWJ!|qJHAHruGz`*!(CnCc=$gz_+t+Eza92bxmh{O zcf9?WuaPrP!;CLSo*c}UAa;fio)g~t$oxv=%0a{Gu-%;*tLzFvsWLkR*2Su_$c#ey zGr+{mUjY*n|FwXL|JM%?wyPJW$Kg%d12t4*eT1@8gt?FZ4qxWX@3S8&q5L%7vL-^}<#BZQ zWRe9s)csFeM35WV49INH7z+-4ED};(@byyrjYObkhd{&H5pbX{t3Y7;~yFp!hT3LjaN00tWm5RRF=ggVW$a6YGLJ&DQ5fjLtYh=TbGdEDeeSJ#LcUaDt z^QiO=uj}bS)xxAD4|Ic3D&M}D0hohB5IrsCp|77R05g+(PA9+0179?M1HKUG9c=dZ z9Q!7m6rf@oXt!X}!%G$(yIID?%RTsRF$Y%LhBlI0f>jY)f$B?|-tu zfNqY|olJ735?LSNf@~|%CxEHeFM!+cjdjgK@!b?=NkK)q?kn9E7~2eD@0@TG;ohmc z1qUjL`=EBZovYw)@JjodL}1Ykx?0LF6F4H&$0!^BFk3DzH&)$;*VRwGK` z=the|hEF~KYjoEK72Z6RTq2R|jh+#Y7GiY`>XC4%_TI2=ldeSFRQmdPz3#au;|en%``yOsxPAgXCdboM-fo7?b=SkLua)~r7Hhr<#P|~vlFmQWT-;!CBz-=XwM_H-5}{p zO@KxGP!GR{L35r}U{07}4~p;QxqYW~@NIa-ciHT+HMQM*Z*>ny9iSWT!6izHnD(fC ztuZhJ-WzY66&QUb&4SkVNavUO#O4tfGG!HREH@vQ^Q3l*q`9a$YS&9v(#Qs#bCSLU z!I{G_?+iD_T#e&ZsuFo|vPKWEXW0Ww*WE`EHTHZ*cFJ6ssCWw@bwx zH^wgJwtJIF_`BjNB;J+S9Z~LJ9&+d zkelvZJKF6)NYoH9@j=_6H;SuMnt4?l#%~0z1`NGLsk`07C)}Emq4MqC3IrFL58f-6 zm#{?Ob~csvsNvVSpQ*I3QS|k<7e6r6Qz^G|KoJ3#qMfdp1%KD~<0RyG3+^ambn?3m>HW|HUb=ROc79<1;%{LBQaw*Nq zAFjy07a3&f+x||6Ftq~r-E(j^+&96EJ)X%YN#?~swYYld)@pEsKI)>o9*!>4H)w;J z8V;uENCugW)%zXX5+8Ykg?{}&*&mO5c6a%gz;{o@+jx+P(zy(O0O&(Uid|FSs0Yd} z6}B@O_6>PvI(eYq0(hI5X{qBpTBCs@+dNcpT1-@7+@fOlmyluXiP>qFllOy7cW=~c zh}%6Urx#T7xl55<-&~G9)ayeagn-t$fFxsqB!{;UNXY zm}`vTY6A6_!>s?YNM(EBG2JXSyR*77|5z+=ZsbU=8DnXz2+tuox_PRS}(Dbb1yvdlq1XRz9B*xdWsbq4YCr1Q@-x4QMDvySOa$B6UkIJjnsGVKE zkT9v5;!JWSBx3Rvug3OuTxU4^=sNQ~+u_FUsr^bVhZC8$Nk?nKlE7q$bJq4m_7%UI z-xOuUH|qCMPo|0ec8R{e3AN6L%+_Pk3RYKNd|W#b?%t(h5NL3Zkja|AEDO6~+m+2; z+r-gQ4u0nqI&JlHi+5vr6P+9Fhp8&Xl)go(OpP@7Q552Joj(Ifx|*uz;uzi)T#f+m zLqQR>1^_T1q>nil&~DY!Yf)WW6+8czX~=99>dAp_k1FF&cg`*%b#A zgT%(MLhy7wrY^+EOKtVQfZKZyEX0k=i^iISt;CC3!ZJo~gcWvqzz~w+K!pYs!I>L# z%q@^U8Xaf_N>Y!IY(4w&%-nGMSN~gxs?SL!oq$&{umS`?A=4+ZzRDm0E6YYcLzJ#8 zo>71@O9YWaqSkF~Dv~L&msm;-6R~!w+4I(qrlnVY3|`Zxc-l z%h!$oCkpCDZ}yznUxi2ATT|iN4ta1=YQUFqAjo27JlB3L z&aC6?Tz9EawN0^TM88#$T_I?ISi+1CJ|;gz;)Is8OC!#I=UOR1^{qZoG);}UUA1W3 z#rqhNu&WY|tuXRT)i?oXdZ!O|FL&-4STP2TJpymKQ8><|U!gGg7Td-=@k{PeoqisE zX~j?FnNO3wUte$7QVAS`jcx^t;fG~k1RuLd$8noNAOvTWQ&+yYms(GHm0cnp=7!45 zi@_H|!cN}AEE#A;8+K^OR*a?)YLXb4Yw2)JY9oX7olschO@a25g* zK^R^tDBo!0XxxcRcO}rFCL%y+UyR+!EkUn9|hYp_)qV&7tU0k4p^=gd6j zoUUcXZ$uli5T7PaD$+TL1&v4Bj^)6sOA^gWC&sGgW6GWK5zR7*Qg@7aCS~NOg5S|q zQNT0CD>_OOc;>cE!7f$gXbBI+MJ}7NPStv6TVA~8mQc>UYUk?QxL#m{F$r{A(YcuB zGoFK%(=<{lq`8GzcyaJ^Y%^3bjBI&ct55{Qn{`>SnZgQpMQudPl*rlWu`yR{D$;_U z;E%p15Yv76gPn4psNbx6T@qLQDb)jVWoVvk+HP^4jIiXcySOl7M*VrP6zgUzWuq{G z&#`%mlNQgYgC?=p9bqb1PmFb*HwjN9(e|-GkSv(GT#d2(Fh6!HJwK#e#d7m_o{s>9 zMbh6=pAKWC`Kkn0Y(*&LJ3)S^REQ?oCL2czrMXwuG28B+i7-VYabONX+{_@|?X?6s z0y)xS(aiE=Zw+za}HAGlLZ7wd!|Dw#Hw9);678J3}7JR$p( zhYT(0oT7o}LDJ)L$Fo}>p@f@ew6+ZSauS|P{C9?+LGWW|r+o4&Nj_^gkBxcS~MKPXo$D;?GNdQSVwDY{H z_W;q0u2GaXFkIGlf8;pJVY-Px_Tk606kSVG``v~yhyTHF$@gi=H3D64>{nTDDs=M{O2<&%j#r;2%8r|9a_=1ez%Ia5@?~i;ID&BzMyc_xX-jxe-0b)!)jb1F$E_I<3fU~u~L9?TECh1D0vwWelbf6E30HKmp>kW5@Z#E(TY%lGY{Q$ zu)%`9Zpsq&N>#rztl_7jMsL1h)Ot~CK@ZD?TVbVBU-fb!sfeT-PZDI{5(But*ZN-> z=O1^`a~KTA<;rF#DJUgg=&k83e@nmip9W7ibmBvhuP6Hy>e$ByVfL0gN>A=o83DXe_%D5< zxu;LLx8fwGPrd!Lo&5Air-*p9>$=rE&<&@w$w7K_lL>{^S~qI2%?~TrXYnZCnBEJC zms}5?AG=6RM4~ zt^l4XP-=kz>O3KFEnTxlHF2YvG0Men-~zGqnB@tD<;l^`>UyI4N7TEs^=v8tEgW_k zOHE}~upF1jJZTgbF{5jtlz=>l0Tfl3)3dS1c?qIWU2T`Qc_CeC8xCiDlQrH~uLxGM zwLyF;1-%;QDyJdPNY&J?6&NZGIze50(&=nf`GuboQzQuu% z@(u=QbM1rIoPy}e#EcZuc~Pi7$l0Y!UoUQ3nG|2*HL3mx?gEo0^t91gA$}BgF0>P4 zSljgDZ!{eaFB!wcW-`@STGk{%-!dKVM-A*~PRd+*n+G@<%>%RdD}j(iZF=6H_9moX zRC(leJ8YyIIf+-2J+;F^Fqw#O8I$(gIalY@O&vdrF_mM5e~C#{PlZCz9fFdobRio0 zTeDGDS68padQEwRVnb0yNf1mF$wTc|49cgBd}|C)$Xg03XsMj+cOB@9+K;L;*ZH~? z?AV}jdM)EQ=8UU_IzGAzmpeDzo8SB^x8@ zrj;yMvHlIq!<2Xj3O*wy5|*VjbZY1oo~cULR2-GGAhsxXjqe~+$({zR9jyp(ictx@ zei!dCyn=oE995jW;g9e^<&Vz|)PX!J{3re92_HNK^Ae4HOSd`;B;3c+)F0#7oFl*( zIS&0W3PKV#4SwPp$po9oPJZ5j?z18#hOg0hbFIsgtxU%a(rt?^N6!+5hlKAeDnvDO z#7~#wEiSxyE_b6B8EP7tQ0?;;6Mt8H<=KbnguEVSolUYHTRQ>o)N=LCMDFqM3t!nZ zh+SQM5AHp*nK0%@8Ilb3>q3i*;v_= zXWlYJCi>2F60*{0aH^L@^4NPtsAxz!wlp-#OoMpIvUUYAigKY44@3vP8d5`marPc{ z*{50c;@B$$8MQQXVF_S;%&Qk}9#UwnXzud&LQ09e+u6#(^FOHm$kan(~5>pvGl2QmIS;h&X$XQ+wL zoIi9Tx)p!@M{Bgo{3tyG4|$k@fw4Joo^}!d&B#U3Npkg+Vk494+_B}@*Y-6-$pAT0 z4W5YamO+5^5ldBk9gdJyvkAxc0-g}lf*_7RSnHMRDVhi$>yb%%Ce}k`9=d&VsOgzQ z<^TN0&SoNptK2c%NJLLH+Uf@{@9zwHFp763jwf!H;x>gYhj2AkQRo7J6p%}gv%*XI zGygwG7XWU&Qv7~%6ZfKM{G|S4zr$xSdxETAwOetYW=x*0ncryk-nfU!^`v20%kzCF z`7rObcY(PqnC{mRK6?py7(K`gfGe|p1>x(N(KN2Lc+7oj5tncO1+MW}tHb)pjPh#T zO3>h~X89CCR6?Hg0&(HuSe0M|ej0K|%jD}xCx-MZ(efPz#5L0-Us8CssAXX_d_BPI zg{<2vrTv9ryQMju<`eq}NbN}klqn`S=5CdiQH^TxT^a`B=g}=+@?Fuzazln!N%CFE z&C``TEJ2*=YD|O7A8@d97Morh8zkmwt;O>Wau1HBs2u;DVP$H6ag6?W+;XGZ%X;n=_H`gyhoiIt&xYW0-@hqq{!`z8E|9q0E2Ve8;=o0z8V}|! zc1P(W&_8>w&{cm&*ApmQ)fwH@P-YR~~L1J1FXup^1a1J-3s3HbR~eH-N=Jj3c~j#ynD};LFbCW|VQ`YN@;a zTu#BG9_v@|<-%7f@UltL=x<3!SBjyOQL5#+cC>uue2IC+bQb) zJZ{~Mr7>gOG-ELf>$)X?NH#ErI^w@6?F?zu+I8Sq6%d3{T?I&NvmV3AW6t-MX&wOr z1wVzi zS`uydo%o(9H`_17M5>y-m?XB6ClgQh{Aw{SM_JYDd?v^ zG#Jihx?V%4{LZk>;TdNZH7~z>_UIEv;Q5>#wfaq<`d^Wnf%Ut zjBX`nmn6^sQOF}zNR1=MSih<>7;mhUQ464poIZH>+n?!v9D)C-#K;MHJnfL4#UY82 z_dM7BO!VOa{GW7n7!Y+>UjvQX3w`$fW_k{6=M;fx=@nIOBjJPGYe4sIVQSK9} z8Kin?>9~gCqZhI7Ne?oxE!XYdci)Tkyg6{v77#vjDs`!2@dtzJYsUYn{-0_2KXW;- zZA{ln?K?G_)R!!GXF`fwopTu>P$Fq*^FqpG7eOBdfkf>nh%GKM<06x|gw`G~oc@1Y zB8-P<<7{uyhiKq`b!W6X)cUDTEcJRJ~ilh4{tktR>619Q0fNGqY}( z<);>yr9z4E68Pd}Qh6PFv&glN(yhLlFE|z{HQY@jo6?%Nfk=G@cGZ>x^{LfK&as}M zXE=uj-!{^yK)i6iZ(iN?k`b3&vgPXbnbpphmA(4lSz&0@jwqNTCp+Lead>qHHv+7@ zC(H&>d=5~4LCrl)Y3|$6EiDju0pL1fXsCR zThx>8 zeKcc=1EJ!~EszWXO)&lmp5ljkJ#Y<4w%auC#otJ;mr|1k*1*%ViLXeg$^9Cx9%6b* zR)iji}MOfSSnV&&|t2&$%*SkKB{3?G`5g6Xp9(gqAZ~Pb{_PZ_B!85gqQIL|Z}8?~R@h1k8$ z5Cwi);Pe$+uohSEbboMAZ(Rl{I!hHI@a{2craniag2 zPtPQAaKG-tKURr|YW#zAG8b373w_CeuKdHu`S%AWi%UHv__O7~YU^9gJHPM-PINh) ze2K$uObzkdAXrh2+l*Sqb~dEcgfwu8Ux8{&v~Y$CKc-Lc4UKa!4gu`2A)5$60LQ5B zDQU+_@ci+MR5Q05)^@GQUY^^pm_h8&_1ST2T*EZ}2k-Q`8Hu4-*fL0lfu3#k5C`ht zQ~Lg={-4j!#lH)LOR6l=j)}O?hIVU~`7Q1MrDY<0KG5_(2p}_~ z@UPN|aex-_s__Jia2e0ZMO|8T-G2%w&slVP);!_+lZoc2DF-UR>SN0jk*`-P=;Yv+ zf8@3LbsQ4*``4#k{ngE%mNMKAM&wXo8{Ow%ielcsXQDKTq}!G-h6Ylq=uTmrd}*3Dr+RB8V9VWUbNg}Hl$U$D{d1QQjUDVcf7(LlZY*oxt>yRUTF1I8ykcgQ zz7-ssIEj?#e{t#V@JiNc6kpWK{}wkm_2Y1WNV!Py?7aK!!EAG%^(wcW*3hNbNl?d@ zrN^g(Zil^3$o?_CyIflDC1W-bN3bp}(WodwzzZx#P8(3B8TtB#f6ko!9RI64|A!3Q z>HmDDuUia9h6nz1J^fL^_$Ng7u!+AA-N$_YK_n{~`Gh4na^YwHchEc?*;u2^{q=PB zkIXM|9Y?8$JNw_7^XG6!e^Nd^>}($DDfzQAH{?~~^P6?LFWdW|iwiP&fTI9sADDx2 zW@FSxaT+o#yoo%I$gN5jj0OuYr_^sVle_pqFB>nn$Dt<2^&utBW6#%zil1`@w7o(m z;KQotiMIUejlD#VO)ug=P@7)yaORcb;r#473`cqs{$wzRFaLt_bRL5BwEpp9vlsre zv~DEzn7ORh+i@v9BDt*CcC|dS0~V{yUM*$C^B-Zyzj%57&Je-3+_83) zwJPP@$3^zMH;hRmue1fHuFi&k^Ip02gl*&er(1=odh=;uuq*zQqqvQ^_zA6L!Zq8}`t$6~0`_0M#l1bc?h>~3B+0@h z6#Q#!*uQn=`I65~Q!@5@MvSI_`LCh1{;jupKZXXISxuLP5u5izfqxsC(BFC!e0E~; z4fFg}iT`6JLN6WX@FwazzkhbiJSP%nmT%acT@y9R!w!n@Y{zuaBah}1_+vk%GRJR0U|Hu#i>-hhk&#?yDui~WAY*n3T+Js3e zgbLy;GyrtniRVJQofusNY)e#)jy*bhAftru>&h~#$7tIjOYR>lFUxCqvU3bQ@#Y=K zWBZ}tX!O}CX7n?%17K8LOBu%b;dRMa^TP@2LAy8K7*?j;z6d3U@J#*A@Ki|R7_CXJ zbR?jzq6k~|-VeTGeVgtIsX_ca>2%Y;|LW37@B_PLx&LPM0n%4=ufxAt0@qnW@l>VB zSii;Abgr{gW%7VXPKd|Yl*uv305M&P07B6Zjk}O)vo9Czn)A8GIWbC(6~z2)BXR#2IpHUnjW`)nxI&0pOEGHs86ntpv22DaX?H`2U?kIBWFd( z`_tG;juNo!Vxn7*#&yHJq7757gSoEVyfn>B76)N7a}$t%rCVyr?+hOEd(!c?TFO4y z^l90G4$>eQ4ge>;)cBgqFkZ&<#ld}<)(%0AJ?szCRa{?mFfWTkt z_}HQ4eJ738I?!t4fB&Mced4N~!Wh`PFL8-FF=0>z&*OiEZ-n1|qT6riBD4XyR}W|_ zxErGfWdpBEb)-J`;CwsL!2GfbxMyVse`6Vdg&g!dtfr%UQ*!~tvZv}$8d_KV)DQ+x z*!1?tsUsU?Mc0qm!U97>%@4O-tX6NnTq{~C{rH0c>sj@kz2`;r#{B{Iq$7(nsj3*w zsfp40@lRKP-(>23XZTI%(5^vzDYujQuV^qJcvWI@yxpomH^0JLp8^$YPn8q$q!kXn z)8pjcw!zB2wkOH5Sh@^BboeooQ9Hsh^u4Tl>wvqnR5+uyzHcLCcpkbm48#+=-wOo~ zM17jX4i6S?UVP)PYwYZd(j5I|`U)dk>tD0~{F1rS9CUMx^HEHm@LvlIMnU$@mewn7&knF$3r?Am~rb*s7RfEp(6dW)a-wKWL^+NqTJI# zL}YE|@mBm#qh@CrQfV&QsZi+)#$$QNHl$6qy5X3fv@*ahD{Ltlo%^m+{^vl3wOO@) zU9Qoq(wke3*NS+!a$vd7GJW>!)nAz_bzxa=-wwctmm6Deyh(zY<;K3InK`W)#=!h< zu(rMJ_=rJ9#Q4@AH9qg&J?<_U5m~#d1*()DH}$~Vt|85`sGeVi1$XM^b-sFA!1rbn{B8LYo{X$$M&@H zZgug{9_05YA&M;o;KiQt@@Qlp< zI}p1Dp4G{HUqSFa*vJS!DBRSeLTtv@wq_uk)qoQ@GXoa>^l=45vssabmZ|Y+^1pTx z*a{Y2xzG(?*r&1%L1{b7@amP&RTcNykj&VQlrQZXRjldauds zud$fXj0}XWL)MC2@#S8Qe6C(SHp1OyG?TJa%)WEkdwdv888MofcyU~T5!Jouir6|| zwH9vUVp=Os+vSQiGLZht27P>S!n`xqbx2*!r(Zns+~Dx_(*u)MgJ~Z=bYiDcoeb+s z5g~OJ8YW-3X*Y6*E&BjU#^}p-PMrUXz4wl4dTZB3aaj>TK|rZW3pFS$bW}(b3$DomEZWZ#3a>mDwS&!l&EM}= z;sHfzT=Ct@Lr@^T=ou4&*0yi-&kAYdFy~w|*wGFA9-&dC(ta{KGCIzGRz+IXQo4S_ z^xU^(l+$AQTW+bB*GJ@P4r$R}0eA77TJs*s$`1Bt6J@c9Tyh$i5H@Sm9oi`lI*svH#vnjUV z%?iDrCGBc_x_;hcIfYdk*S4AqJbh!hDmL9{5+UXG@n!>>uBmLIphMxG5Al%ZkqPc= zp0U@G4S_ry0vNMF=oEJ&QKeBaXQoOtbhs^bjO?o&FtWN6C==*ssT=(8h8a3Kv(`jZ z?5v)+v1*oXmbetrKG-zRXMgwDhpE@-={EEoo^FFGg6NxULm4;LGKHwk$HUgz6ggF3 zJ0`Fz%!E4ZGm>jUn4)u$r6J}FP~cSM)@voIA^V()jJ^3;dY+YZ68iDpJNiHvVi>+= zR*+k$dci0zSz^>(1Y?r}Ad3&%y{2hDJz*DQQzPk25^^Xr2)~tYae?}BiO^BdriO#6 zQg>(twGRGtSermtt}T7t*h`DdE;)U>c2=;FuZf_F z062d+Ff4DxUPa+XyX~C~ND7ed-XQB3pd1yb!bhl~u4lDEmm|ugQ=)xu`wkLAIj?ju zh1}Fj_MYyaE_(LF{zg>iF`sgo{p5h5!qLi;>B5Y%PDu`Dp{QI8uuahua*LagbK=7` ztL!2K-Bc(~%I3La3)?vVl-XUE^3+l~WCKd(;Y8c&jF$|h>-(1(f)~FILkfF`hxlW~ zZHrZ|9>5*zF{Aas-xN{fm`{`;MkP+ZCn4-MR#8bDPjEsmg9D!FT_H7SG2JaI+%NRkFBy)rSI=L*$-k-x`+WkN)JS3DS!;_b1EvZRnBArr3{eUm#l`E$_c1+F~~9?elJvx$AKOX#vIMi=$gt z6Tm}n;&Y^rI28h?!|=e_9{7?Uu0#j8;m=eDlvF2ASe;zo2BL`@;3k_)W8)ntCd+s@ zmw#-wP-GZfEUM_aZJLi2c)dQ5hLhw*ev+>Ea@r)bCe;s-a?#h3&gS_xFDr^3#sX6tir-4!f5m?6`l~(^-U)uSeS2ko1(M_X#Kl}&wL!Mu~NH!n0mcFdF0W2&Uek|-< zBfWlo3&|eSw@iX9FjaBs`3)99d+8=FQ7?PqmHv>HRVf-K5{zecm6d?14Nd|RN6$EP zo#sgMrY2bL=AEK&W%;r)W;t?4a7`_32{b`%0pv(M#;s4Xi9#eLUb#B^$6x=I5k(Wi zqzlzOo#v&628?HP(1Z6Zb<<6sw~jcQIJ-%7vq-jqUiO~=YLtU zT6N8xj8R?uIs0AxcS|m6QZw6SkFFn0U%grW$CN2>x%apW4gJ=gWD_&vSIA+YD}LO1 z>uyX-x^XGzvvsnWGWc1Wlj6I-KJDL{*rDY8sb`YW!6E1Gxv9NAz~zx~PbwrYb3*5`MNc4oH}NHZ_fR?rWd za#s)$oB9IkDX5sddmk5k|&=CQJ}~Y z_J<9Mdfrcsu2kKH_wgpb3G+JH!~3;~e3v$AHIpaf2BDC8^ovTH!LT-4EcTH#BV-vB zu(^y9);P)o^18kOjhWY#7Z+R5D;aF4bXc0TXSi6W`=gV>`FQ}N;cDS%wGT+nJwVk) z6ig3o7yfY3cIr*b9fgwKiuHo4Rv#?n3#1N*r=79@c}Ex(1W7{QcI_P=BPOaO%J6d6 zpm~&ZI2B*K4OCC&uhKLCkjvb+uf@1B#^*}(03&ZSDA(CfjC5(6zkxy{#Y2{BIElY+^fL`yP%>&K zr~RWBht3|VAI8j1-{MlX%6pt$bMoh=H}#0y z4t>9+)J|E$5>O#y$j8YqV4hF5zkv1h9MD5Q;lDt@6ZQ_WV)5J#()PKyizaP>N{d>=VYDj)t+i> z^{UX*zMAe6lRM*%w6#d%ZcjWC4^g$9g&gU*qjmHKPpyURg%>(So9N+~wY=>p zL-NSLq9>$bF;jKED(EvqhJJxLmc`UQ2W zxfv!ZM+%s$DQ{lZc;@P*=Nsn8MI$!iOs9rh8dnsF>&dkp()hEITm*oiC0L5pNb1Pvm^| z$LS+T2@A`K-+sZh^!$yLJ8$`S0>hB_ciMIFtwbezB zm1pLIDzUjP%|?=fib?>cS-mGp9|eK%$LlRFUOW_Yn@QUi>f`96Z4;cViouXz4vJZt zv0d90w49G4@M8Ckhy;|&=OgUY@OB)k?JF{gm!7mGrF?lLTdI8cGq7W+={wEFjj&L> zB(E>tTf;x5IaP9O$@tYPTaBO4O^HehJG|JUi%{o5!z;l_7)O}&u&9{QwAf(8KsETc z%75Dj|E2RE(2^1VC!XP@zdS?1|Fvf*8K`eVCxYLM)3nO(kriy~U5BM&nkzDIjdJB=7rjEUxqWEsh6x%h&q%! zfGdh$iGJ0-)>zx>JX`;>bT_l4JBbJ+YD5Gv8TWQ# zCB)N;+3ioX*X3yy^9~hTRAkf}v3P&D_s65BdOf?w%~@?ZqVeV(vejP&q6Cr_WZ=ay zT{`?S+>^C)C#I@@ybsi#F^1`XbMF}MIgq`gE&a^^m5m;w)a-B8;k%W}NfGZeR$zc= zp^)|$Ez^1en->de@*Wehibm@3&-cb|<)w?U1f02URjU?!vY zgJXU>9h!jlXHVl)u2h0-6H{LaMPNW=RWbPXN&YlNm4U<(RefGTaR|=yoKufQ(2_)TCMsDtFjFIfD6Jurr114uju=4jz2~DE2A?%(u-<%qKRrxF>+Nn0B6TnI zmY+OXMhX1c@3?fe>A)kaA#0TiCo60u{WI>^X=K8#D3bK5iwZJ!=SNz7SYuacYV}yn zLv~3kCrTogFC6PsKkX7kk&l+iiCMFZF-UOzLTnkeDQ-=^N!5I#!|YPch?)+wszNwH z;kMwW3%335N*g23SC585!=VLSMxCiI-Y8;q z&&2B;9kz0vgaE$OmGWclDZ(v`wXT`r%3ReLLQEh^Blps=Y*Or`Yp~$znIL`?4|c)ku9S zcP0EEOkrn~ z{-G4EfP^)?y>$kLAMk`KRIm3v*fl99DQGAk577ImeMWTCB7x<9Q12Mx<~~ zdOY$OLpgy1@J7*u-m?4uxGDUP(XYw=HVY_qAXGecZ|e z9o&pvZ99hyv39u;#7FNw;bA4^zyJSY>i;ZEgBklZb0uU(TQX^A1cpIk~30A?UbrpHP4{t zDGV%}cf)m@^&@XeQZpK$+8KN9(&nIkCL3HyyZw8;yKA7gLkU=-1K|*EUM0_pkjC|T zDu`<_V=lU@bqT*8%e1V|6H{dD2&?+ts#|nm0yDRrgGxt8^n1MyWnrE z``rDjk!@0Ag%(vdyxZ`Q{NM~f*qd<>c(U^K!m+{0*k^M*~krsJmztm{(- z4|q$(V^24AgYimYtUo!aF1D9stuD#@(HGrzwq|i4K;^Rz5JTZD6Vk61WsB*0s@LUD zT}+Zy(z%dS?_3(JnhNU4gig*$E=DWyI$<5E&~Sdgi}YBsD6xCd$X^ab`|2>X$!;t` zx$#HzT|FL?dy(6)n#T-}X>tbU5fy580wIQYkLXSWhjd+gG8O^x9k%^Ys^{NE(a(e< zIhR4-)8aN_=<#b`hqt5ZxNYOu$^jBJo3%zZUcj@1`&LOq5bg5@;_-UNgsVZ`U4F6P zzQ!T}vow0pw0jjJsDhqpFftWjZ>} zp6xj*WeEm_ComC|D#2)x>N=f-aIO>qMRTr%BdPlMWEn*rJ%te>t=9%ZO3pklzM)`K zPc`PSy-2Z$V1OxJ0Bs>a>dbW_v)Cg4YCPhWNOf(ln2l`pV`j8NX|_X5O^gQllaPg% zOxb4K3TWTBCW-xL$h)sz^RFlNAKNcJdo}J|G1yvo^RK0TqQq(IMZ3WEGmoc&e;hfc zfBR<7`8TK3bw0Hnp`KoyjY6lK?6D`?Ucf}5<&)JBPGoY|w6y~U${tw1%@qp>SjqYE z>ie-kN7>a#Z>HwWd%pF@-Hb3xdG@=PWF^c;uha!7jyZV>8D|t20ox=irsXGQyyFCE zN@GeTjHu-L? zyh|~cD?)B+7bMhVqUR-qQ3C4Ojdkk^ycyVTQ{Sea|7>R^_9B@2|m2Fj%1xn<2AniIU|a6H}Nrp z_uEXoj;4NSS?DNC%?MIIUh9jbIOS;@>e{61Bg%SO3}OYioJkj{xn@6oN+*X`c1GSs z{OBbU=eN)k&o;bVwftpS>>gP?tp3sznFJku#oQ@ezPRw(z`%eMt1>>gO#<4xO?vyE z&I@f~%6BB}R;k5t)~FuIw<|JEfADR%CYQ248Cs*&c$hf*dbvCqUf5g`b?Yr)l9Ddo z8}T7ezXi#a2hWx$v0MflLE+YHA5?b3ym(m}=9fzPD?mT%`RJv!mNE@xBa#IclJU3gPZ=>nT$R`5iot$N~XVHO=OfS73 zzZc=*Z+v1GnjQ7wb0K-gWYd+-yLI!^%=LGUwFAe?&SbIOy%-o+H3CyLv%l%x_xbK! zSWn$=K)TBH9Z9OJUWKzog@Hp@_bR(_O2d#dSFSsz#cpQtM>iqXR~KK}i6xG;UZiGtyExpL z^p|Jyo-zs!V)n|HU9qHdSzeu>{RO+rqvj% zS(aOim>!k=5K$^~xC^%P68sJSHVpnb&G7C4pA<>_>Tm#;QG>Vb@N#ydtUBmJ)iwM4 zEkwS>$G3%+ZvLJ{eoL&LLpcV<9wyF8edPSFhOr!6KSi&_mnh0#T-bALQ^&^49AmT_ zI2am#V+&mgH*8tPsomw&;`mrU?i)B$+Cc;Cjdfm9p_tWr_6cZMh`w*YmuQdyOoymp z8TJ~5Rxf?f>Jz~B9VDSr2@p99jls`far({Bw71S$3mGW6P?z%K?$k4RL~)I6ebvp! zITqro#u3S%OzL0!=-_xCF^1brLr*N*g;yhL_rpPx`FE}@MfZQGux+=*z5ps#S^3%a zHAliE(4MG(&cVbIcg@E4kF75@)k8$bwmDA3R-t6t1ReT(za~|Lwn;=o_X{V{gtk*B z+G9C7N(_xZwsatk1`CH>m!`$U_XfUA?$W(zQ)@jIc1g05aMvR&y=aA>DpZa8Y0Ohc z3@Y!GWKHa-VAd3q5hWf=0-{{%d+U|o${USkM>92$D z)#k4zkU_>ONs+pT63S*pgHE|(0!wU8aw+k+;w<@ArU_$1&|AkO#bH>jzPCPBB#jMz zl=`@3nm-BN9&=Suu6a31k-clKM|Jx)VqyzJ3d3ICe;ojV2KoB71VO#f!dTkq=a-Yfa=!upG z9hUV>FDC&(ZiAxU82q<%t0-vokiJXhZNIcP`zl+fg@(#P8f=R{{g5*z9JsYnK8d5Y zZuNNPcO=*Aqn@xuFN*05o_!uY+aQ12J$b~7STY26%E+ozZa8E! zYa4%DHd-MlEzC{BO08Vl<+n&9-QgaruW4-!6=NG$Tf;}eA{j?iy-bxWuGJzm*5u^s ztjnFM0d4}AjmK+S26oHk#oUqAOMP+KKI@Kz9yI>N28pCX%xEe#k7(TcV{-USZLwcX zPok>i;23PQ{%n#)Aopk*wn6%9Z!N=HbV<<@t@w54nA5SGIU6RYBUUCo@tVM9Cf6)g zIt2bf>9uTujK5|dK*>!4`(!MuGqSpcLzpex;j^xOCl5zVGvQ3Z220IOSOKTla=G81N1d@72-^O&+^$A8oJ5}@vc)$ z6qKDYA`{a+?2;Sl&Y6F3{#M1JeNOq9WTJlbvpG%*wsyWN8=@HZJ;Vg1uF@iab!LN< z?62cFOTUD@hn&dMW--6%h-(N-_G9j><%X^Gj}#9f(|A0dUu;S50=^p|rOTQqCi~s%j(%IyoC5`68+?y=6c3$=!riJ%zR2PKaY`%3UbJt_L<}=k}Ivb`>=Bh^acdKj4hysth&X6VNc(Gjk$)Z#FOobxzRV?o4-i?i| zU5i&~)JILhSd~O?L_EiOeU8I+6Nv+wW_N0g{ihQ#HtP+bB%hq&VDlBiDGu{bv7Duv zi@8>-kpRP{+&VVBPHocosG;x>tzt6B&({m)Lf*3zs1wioFDWe7!zESHFLQI$cxR`TKth}IV1;i zBOY$L*>J`A`|tjDY1GMTKv3dSy-Ez#x`?2GqqTAUCWGTlQp^^w{Sq(Tw4luU%_#o; zAOkEAdfYX@_nG9iOZ15Cx0i-McJi46RnCUFY2~JTwFnD4zUt<4Usd*qSa8;I_!XPa zin!4Vuu;G7lYM-m_Y{|V8gZ?{-3F+_SId=+#PHGYxQ@V@92+Q@P2eWNPXq*N8O#ij+C-vt-^n@oNh(`tk7h>-dyYCWVW~)mb+22 zm8QHlmUKQgd$3uZ>|_5{5)Mw|dX>EsCQyhST<22lOh;ZY7KQK=$iknxmjYui?fExs z6a&YrPrgq_X-T=?u80XMN&v3NV1t}xyzvW!j%fzbXx1Er%pZQn^WxiNRa|Wc&q9{^ z&J~2lMjdjwKjmTO^%Zl@hjv%a+FfERte?I6@zd?MvM9|Dvz_~G%&0Hb4Z4wu1$0Ci zynD{UjaYC|eU{HQ?=|=SrIb7b18e`?=YKU8b6u-W%vrYDX zJ{rO9ZJsUu3Mq%F{USJgbk5+Ezt6~{gl83?vx(5QMFK}_ zvy(ro?0;OE8hMm*`@$;&y=FgB!}d>IDr)W9DZN)oHl@=Vi z2p<9>UjsA6W`I5dO9NhaYGsw2q33|)wXb0*K9+IsN^Y+bG0y6@(La}Ib zJlV+!NhEJh;Nt`52JCIX-+QcUY@_KLJqCH^)6_|T!9wG1>E_mJi4d)o?nnBG;$u_>21Q8B z%9E(t<6h;#j^j!dceV)CQ@g=3%0ynFQ6V9Om_@RQucoRJ?q`Yw8gy zVN&y5McDUj=3Myoe?%qxtKolBlJtB={NAHyKMkIJJ3aX9h#l9-32fNubpM75@SD#O z?x_N#%NH>WSv^-L?}}faSs5O9@+V8=szJ{Wwyzf&ak8g`du7@*Gk+Nmyph8Z4_4}6w~>q$=-8V&I;kc#gOm%-roSQW_X)Ary>Tb?juvdD zca*sWt`d_=?7RpY?vwe9OS<^TYBDs0S@@eS9BtgoAMsTJZ;>k@sVAxg^uOsDTZ6~! z$Bw6$cK1Imco5*66JScu$i3~#9cr-(i6V$lDpM4~Gx6)O8~l2%ON00%^Q;%r_jgL^ z$vqZMM-nqCE~3Qldy z6ICC}Fsmchid5^7l76@HLV#x?q7A&x2R{zvtURo=X`=;u+_1(puSMpxOQ>78_D+%V zrWuP@8F|KqlZiz;ii{ICTtmp3&g=QxMv?t-jkdG$vnxY&#`oR@T>9}s;by1rba3o; z-=8ej{hvHze5Y$xA$LGc4TkEKOF5XhuqE!)Eu&-&bNrcoRseNz`Ce;dF{q~E&Z}W} znWj49OskiarQyP^FlOad>BWcc=^RqAF#e&5qCw}zEImdTu+b*Qrtwk&sRL9mU#LCf zsT{F%+ZvEjsj7CA?%KD1c_r&I>LdN-Fpa&wG-~?9)J|=Rw!Fn8*TuoX)lEHdJ=_xC zW=^Xz6^O4^)DF~rZBdI8c)gbde#Ad2{Y94ZvykbDsTc`zI}~lKOG3L#T2R`D-?JM~ zNbm)xXPj)Nc7K?eR&u)EKSW#_=JTtx#Pq$kugPlw*Pa>4i^LO|Dxj6+(2Fb3| z?~NXN9VNG#+%k->wk)`v+oU36R;L)`s**?yExZN@*47xKVUnlphY)mWuQg@;<|t=_ zuedsyv-cnp?;nhEQgc0}A=i9*9!Es%4xYOG*INEBr8WO`Gx+88KcL6+-?8;FY`VuR z!pudriu0%%cqDI>_>u%fkr5Mf$;Bs-G_c#@FDze}MWDW4W<`sY;D{ z;(e&PS(q~doZ5|LGABcCdM(x6B~J4$8blayoro-}{hr3& z@t|NLF0}sr*ca+cd;!-#WUQ{6qV-%Ip7L>@50D?jQA=CHqPYDVyfq zsyzA=1LxbWiBhHI@4KgR(;D+K%@Q^b@R0!hN^pc}q}Z$qygqjEo`7J<2c~senC!oo za!d9V=xhe5Tldk*lunyBMlU!fKVf3NTrYmUGG)o|5dAu_bPqJu*ee9yz7`v2`2|JKLWA|+D%1ljtMr-|(H?WM2M^ejpliq%>T0r2xk7K{w^t>VW zm9h9*W$OE&>V0TI=FBS6FA?@09L({GzB zwmw3PRwjgy<+oh}-{Lo(5B-zx+G1LTd=}w4YtBhRl#;SPF-!$|&)f1QCtLK=-q?(@ zf3%HvuQG_$R?)Az9c20mwxi1xLIp6vRp#Omb;j?HEv%|0K5_FZOUKxcf8qGaVh_EBaZv@4btdQh4j8z>7wk5&k0NM=*F~Bn)HUw z6i3jLNCM&V(A2aTq$p+(JBk?v+vbz*Ggf~kQ3EsGZT9|T@w)Z5a_+Axx}1+YUOk;} z6YyLmKW7<1KmUi5^B)JIwD2Yk$-bfHE>XrWeWK;Q^iH<3RtYA1pvtOmc6Vb-iucB= zouOgJbfrg4fq~s8nO>caBOy`s(1OemwbLl79b`Yu1C zAQ>GtU$pO8)yV4A$>lsos8#_0C*2ovDC4Z4WU1cxgBY9fV+S=S#o2TxC;E28`Y1q| z8LVs)E<&Gp`Xg$=w7L?`8%&b|Lm??5i7C>2th_g-6l+{Jj#mqn+|FHeixkcDx@jpB zeDi_jadOK_Ah)n{k=1o?oSdCwR$c%e>#@IAml#8hKS` z+EO-F#0{AF{gfU=LTC2N0BE5b2LLypWE=SwYpu3aV`qJ;*V^iOQ!y*8EP66jZIRWq z14BO6d0ve;cZ>hg2t~Y)&Xx#S5>-6Lc&^qxH&r6Kq^XBC;_}>>$L1b!>&JsBR8Fy- z98U_YD6esi%WB?ap0GecppQf*G6lter}Hl0YOqs*Hypn0>Cq}J!3j&Rb$_?*JP+^t zhVZ>~*}2eK)VMTh6MdBt+6bd+0mzN0;}tg{UVy-oh1KuhbU}gLs^T(%5P#odWsJ?p zSdtqt54vcIb~0jgf%7>z^ADZ7{Y{Q%0Dt5vRBMqj%tt)DNwDvSD8!UbIsT4?@Rrm?AWt6rG$K_`AH^bJdsum)ze(On~`{Oih7DsI6N6pzR1gPqL{wSddKI~p`aYn!#dHY+?@01PT@cd(f>AE)@y9 zRVAat*y>AciBS)?iKJP<=-NKaN~ZWCA6aoE*8F~o2sGpU^ex4y3<_=-CKV_P@f-!dAxiLTcA#WFl*MLI{U zZYDHhrd^V4Nb*HwCFBt;7;Zv%FmzLVOn8X6v4YQc(5Oj#Y(s~lg>)j%E`+q&Uw>|z z1oaOs|5BtqN)gYO8=m*@DoofJ!7v{>ezTLQ8EjYR+LZ-_0+*B7x_O#I;WqEAYrl{i zx1H36xuFtjVHR0$dt*AE``lqY1)Yi(T#$`xrAhu#Hn!wcl;mt1mD6HU-LveY*nBR6 zXgvEcU$xdL=@#!cF<%0Zv)bOEk38VoE?Y?3qYk0$iPHd#x75Bkir@dCy%JPHdGS$} zb2}Z6R+3aXDu}+yb6aqBZ|3nV@Jjqeg=W+E_nI4wIm%| z1o+vRQX3Mk3&!2CaiYj^`?W{bS4&^Dag05mrT`IxO|8W3Oy7cZmuD3c(n0vTGchM! zPuL1zE7jA;wabye+d2loxDc_>XEc^dfK3D#d`ek2tk&7|ZO8$y%gU`x|@8 z^~Wu`a8D*m;j2j|oxL$Ng9j>NoQGykz1}XoLNd;n#YaU+D!jm-!p3Kp^ZVl5G=~CT z5fAX(sI2qBgNFerS6N{>D5e_L<1A6LojVNIV72h}ymjHwRm`*mBdwSm0tFcD!#|#= zx|PO>v2V6F@rISf$!QKa2SsMpZ^$@XdY64D^0JiCuv0~EzP@dnm|TXdkv^kj=GrHI z11vEgh;(=J&}z^rw%ntty1kD{v0c`A;RSc1^!4@qyc8s^A9&bsl&tKQ+;p>zi9T~@ zLXl_a@gKo)7^)R@Y7w%oY89zM7e?C8s|mB-+u)GY!@|zhsQ_=fIL)bh4K}}Y#Hu=z zLK?W(%B3qd@6c7`weEb1WtVUV@J~zbUlxNuV%6+^*TAK1nnU`UObF=|8%WfhZX#+ zab8pY?zqG8)XyJ%fX}}tkG^IpWF5QQac~LErNBIp; zO0<(=If?VNxA85v@wY?*vPI)(T8!QOilDyAS(PKcr4Zz(wWF+1cVCZAUzZKwZNaSKbdj~IUz=xb^+BaQQ224e zf;=#^?&m=zZTC->M3#0NYx%Lu(d>74Yuo~=Q)EgB{`BZNCjix z#yU=s<)UY}YLt{1-*`h6^UQM*t2|K}D@scS=oO27VZH8%?F29l*4{f6S4-`C{x8SV|3mTO zzvIFGRMxJ~RyA;(-uUv)&#DnaNSxcR{(^&Bwle@$p+oiCY80%VdwG2($J06>&>Dp& z4~vTIj6}YD^-ac{f_}G9=xrcJkBNmGQNzO`yi3RFus*l(hXSCb4hyoXO;2?I&fq#q z?F}rfc01N{R-Er)tkiIl^3gKSuaUo%`(8%qOZDhn$~F&H_1@S3kH;!C4KqUhR?+tR zF_I^U&>0{fHcc1wJks9TM*1ucM(jwB2}KyaSY<1C;hlLuk1+-0mgL!t<5TKU*6c(Lp%iSVpbZ_D(Uj+2BoDTg{4wmG4S!B zGY`_vxl;qzwHAtujkVd4+xcNg*t^BRJ)tP8(6?*e(xr^J+G%|uV6a|DT$ty^kHu!{ z#$tYg6Ix8$xcY`#WmWR`*s7*4IvdTU2XW@K#5BDygNMdYffv56oq7WKAHu>SA|}ny zWIv-B9rGbo`vJSwLkp%KRcEcf*sAR~$2eAg1f$Z0(9qC7QVPEtWJ(2pVK#J0UMh}= zhxr^E`$G*mP`ORiofMfVDo!_Wa<$4Y;N@I|n(FoCvv^35${TI`` zCTwk;Ot`H3`W{Z!-ln0)`gT``IK&ZT1`A58}t*~?zd-I^MD~b z?0&}?B&Kp|F!Ic;q8>x8P^x%9{vlNHXI>`j)J9>A^LS+LY>Fj#g8VKk4#jh~PR3sb zS?t9{kyRz;*|x`Ww#Zt#Cra9%iO|EDls#o0s;(!8zVa)(=pllT!DBjH)cHuv&(a&8 z3Wa5d?Qk?YV@2AXYq&b6#yzVrCw%20hwdN71Ip(LzJ};g zP%oji!buddwW*f5BVCi0A2aOyLu2Rv1&>{D&Go^#Yxv4B&(cLVJ!gAMz_kc-{xsW{ z`Wc6Afbkp(P@*Fp9(jUH)yHeAAmbMlhID_`-SjPVt9U;W)m$*r6!B}n_&KXJ(yvDt zs8+Iy)0~LYxVOI2Y9su$U=d5>(GaQ~6+V6e7wAQtkw}5b;WPald((is)VA7ym+9u> z>A)`|x4tR=5)R2Sw-`a2x=9{UN))4umHZDC(g94~-89K-Q)6UdpiCmHIk$pK(5j1F z@deHGNr=G@limRu3gy!6Xg7&2=s~j=&5Da>Sk4$hlMAI25dwe8UMXv&J`Dl51A=~3uzk0 zE+G<6*Sqwg=BxT(hyCdL>xn_A!)uhFae|^RCd3PS&N8zeQ-g%nS68EF2^0@^5FLWS z0rLq}H+%>wQX990G@p78ZK>7qau^nT$jD4~M(LGVNh`;A-Ym1A0IR!zMlYBOxzg3@ zLPpBc@GN*@J&IhWtE2@0)D)~^YN zs1HbwJ&losuwg=$TFs@LOMuVCzs3B?vOq~Yf_6VDVwuMk>?O9*_wKAA8Wo!q+bgAF z4AT-;F9HN@c)XBY3l?_@H|c^9GF-^)F#JtoVo~RoiC){GiDQ6|-Nnq3vE4dcpG=^) z=L}d2u1k6#ew*Qb{XK*g*G@2k8k=^cnBMYW5wx!fgwrfuI%K|N)|Lv?^rv-_?t>Crnux5gGc0`iY!;&k$<;cdF%QGUU!ZbG|*9P!bGsJh31hM#yS( zOwf^CTE6I4ozbRUrZ?g7KT)V>Uj*U=qirV7 zYnpZN-ED3lrI0?V!trM?v2Z+?OnS$R zJgQ$=zW30{=hFwFG#&QZ>_g$u{Zj@{^6GgivzCuonk^82d+2}n{+G1txrV1E_+%>L ziUaRX@?v&+Wb0%fSyt)$$&8EIo;G!wz>lCZ$Bq3p$XX2@^+@5**x8%cs_zl_hRfo& zV(~+Fs=a77-IrQDS9qe!vukDIZ{}U}a%@oWdglQU(~KTH;}Wcr(g5KDkyCCtH!z*+ zf5HQ=s*tvxsQXyBmfIE1Ru?`#YcR<&jB$As#Lphk1$Q>U?sQtSw(3;hP&$DV_r+OY6t7O1?<}gvm9i8zcso zjDy}jTC%d&$h!kGZ9v`ie8iXTrH98@gj;>iAFj`LzE%Z~=z~Bn)uT-dCs>Xr?gh7A z{A|@V*=*4}jxVb$vKbLDw#LDgs|}`cE{$DELV1vqKt09)Ty*1{>0qxC!^MfHCjJoJ zt#%+Dt9~Xjc^S#k&mu_23zGWhCN@^FX9AE4$!`k;kaYM$s6;T-lmST*&*2>Mt@@lh%e{?%Uvlgan8^e*@#=6n(6%_@RRy3B#8ykTZuR2-N z9~w-~!==qFG9d>=C7qxP&Y!pB@b|t51+d(ZQdDuD>JH=^6IV@E{9xEyRdiBgDTl|a zfB}`=AlYZYSNhUDFF(NRMe6d-mP)11Nbs#%9o|!N#Ij+|0(=_e{lpe}LnBvqFee$b z3^k!E9GKe2GCoO%bWO|ToigIabSJfsiAZs-~Q`sHH%jPp&+I5Mct>QC@LV( zSkYX10^`{v1W9pN6@*qRK~}-XqwDQ0rbIt5OL>)Y`%#u7?o;*vl)kUQ(|p*-lDk~` zLwr8r1-5)nd`sdj-%8QJ&}hZzVGOQE_+71%@1)PzWZ`h}yl={^FCyX9?X;jqbLy_Dg~HM)LY-j0-1-!MijEzOv|{v-tcFI8f* zobLMqzvBt9iW28LqU1k5czW>k$Jpbl8xO?pFWWv~DA}HUHJs<;Rb*%X`=F&!p9P0z zD9~w0Erl0vp4fqaKp(OBj5b!4Pt#`u+f;w@P-oQD(_bVJup7+X2Jif#JJB#deM4dN z!))EN;8{4-g9dZ)o3x%}lbAtVVyxrsH7vHo#0}#oK72pp$a!?HMZ@J)a>F zsF^0~a|D6QkoqTJFnJXX1$Eh+)W?0N z3axUXYOrkxmcqAZke;(>@+BTrU+^F%F^R8UuBBYYTGxe#Uk(BvV}uR;On_aD$Z(D9 zV=rZAGmaP)=u*1AcKu3*53pRZ-?*c)a_Ne~jMb96Cp2BRa^-6n&MF25sv$fcmSK&c z3)@EOKALr2l#sDm&iX&td+)d=mvwCzb>R{Pl&XNBl+c6HrK&VZLJds_MFau~BoygI zEcD(X^pXGxO*#ay(R&HKDN>{(NE6ic#eKfLWW8st{qA$#@9g)S-|w4$GS5t=J~Q*o z+;iX8)qWS+SXtxWwD~O+*iD=7xo>EmZ3HlN<{M&*Ay{ocZ1n>3zPm^Da*&_|1EoT7 zz_TCp<7Q4x+!3vrQ7i)0t_%|w{Z<#%cnmR#dfsvxGNQxSP;G&lP+<13HBurw6i^zV zqRc~H_)p50*#5g;k`XM^ZWb5{2HnbPGf!~4-Sl9VErW89oqb@VF)Da0;2vdDv;C_H zM8l@Gxt4MLCgd_D+yQ?#u&T-b($HpD`7{%f9onl=AjrT|l^KFjN^8uOlq0Gjq%xqU zH=IoDs4%=%yGg}RT)UATGJMnX7!BufqnT# zm?y;&YiUdG@TYHI!b$PwkzyMiaTP$t(h*L+p?b6i8sdNtl(pgHmpafmh;3>r4JF;J zBT0FcfxGudtzPY1g@hH{tK>5=xY9cS=9c6LjZ2Q)(Wb3H@Fc|DQn*~^=-Hq&Ng-ny z87W4|&C&aMv}Pz#)|@vr^isTMgEVvmsxk8gO5p5=VJ5O0-B*jc-YHqq1nxba~GqJSMD6O=S)*<*rktKyPM&_ap1b zHv=qn=9!b9N=g%UnYYdiY?aKi1W_|@+bb_UGNxYk$nH`Hab5$HDctXXbN3q+jU{vYBc1tm1oYKm*GF$S!Yr_0D zre>#%hI>EokEsA#12Zzm9Q;ELy6u2jEE|$r*E5;vhQN z4?RQix;){!#uTS6bp1v_CG7Yu+SBj*EQ?7NG5_ml$nhFCQ{s2}`|pl9J{^B+6sx}N zruk(l`CwG-vC;f;P6DdvJ%z;O*FU#ZKYaf>TxYA3%2G1X>^9-Hj20}&G;iG-L1V^^ z+y>VbG|P+J)Q28!KaOcX7d+SEmNz)Qb-mliGqQ3xJjOBO1dqlD1MAIlRoy)HqQB>BJpGnvC(N&O!kEfahKivHCJhLFe`w%#`2(8}*(d4`Cc2m05I zhwoQ>0NZFPurDYfgitN~q3S03YHIOZWU5S23;vXph^`5ZmKM0q9SVa#pn3m&u>5iN zt+kV3)?mKfGM2Dv;9?{4)Uz`YYQ_~VR=DeKTD!VE@F69HRGNAVL7hFUF2?OFyKI`* zpit*Qh5$=Gb#1MvWoEPUaX324k(St~IlG>`T(i8i?GF~U`~PWAPSTc~giX3~?6UDR zrgCG&dulg4NrB1TuM&dRgp+ZF4-p!gC3T-sxsPaoy)B>zJws}7bMz)_9k+`MjL>6^ z&XS9=iKZAxwz-&%ezJOQ4?%%F8Fh~hEt#B<}aL+h+ zwWC@XC>E-NeG&1<<87TFPGF4Q@g7{LL_io_Q?i(v4vH0Wsv`3l5A2^9^}#2qfKcO2 zc$j7E$z~T};9{N1D@`m`Luo{M^fzeL%g`sounlsLf>Ct~QK#?bc^rUYcrFDRkghw~ zLdadbXLOdW{fd^VcW2-sy~!zUI$14H8_ID!eJ?U}#m zCW$--u&6%^jMnsE)c38IR;+ANFCSZ1*=++jq?A{=GfjKhJv0D!9vqR}RkFvevR@k= zqv8C$cj~TH5pw-)(%zhP>kBk;2AZ7~$)8XZBoEqTowZAmaFxyCg2rIZyLL1%u_0$> zm^gk;qE9F077c&PP=slN9JR%i!gvbJaV4LX+~Y5;Zj+IxhZ1$Xopp;o@x4H}$hX(g z-*RCLz+jIh=<;HE`jz_T3rc%}kMEiknH?zLojY(3S7MRdKqDjGcvaMw-gr?aPH%y+ znQv=hl7O>zWD@mIwl1Qnobm%XV%)dW>y=xnBO)NW+voE_Nkz7~D5lS@W@Ec~I%oAFFaWr2SCH-j^kM;X%)B zAFEJiUn3mXbe#}BjT9z)ZyJ^s77;YuQot(BRELud$P8@EkfXaxyy>=W+UI$LPr>yy zLA6bvxGT7e4}H%&oRvWnq;srM@$R%q39bY&@=Kg9I(9!C@L%>>2tQ?Ts5d*Sy>YDt zO;twasmeU-@P9i8^78I%c1TKd5A0*>GYIfVk< z78otTWa1&6nvK%`3tC_g8T=l*_Ztbi+SKY47;x(gG@j~=6l83J-*f)u7+vKp z$KC`&;dxDBllj|DXx++IRON>~0$JH>GA<>nJo$*`Tg+ljPhM)UNIet!#kQ>g@TGKY z>8Z%efZ8y2`ZF1ISxLS~7mL)5$9A+Q{%Ud8GiUI+%+${$Pd+?0-o|%6wy*w@ExCL7 z|6*bm)3Q%-ZNE{B9|XVKm`lVT6CC>`UcIc41WVm2QEQsBsC9=mYzYR6CCbK95*xiC z)NOFspqU?t|7IQ@BB}HsWE4k7C#1AEjB@+?22E8Xx3HAgiV}v*C04#AtHi64VKV>V z{&kjeWYkYi!#|Y$U-IzJvDGj@;-%YjXC>u$Ln9qw1Qbh3-!OY_OIW^D+6puc1%9lo zhLkAqDdz@K*Sige2s{9onHJ58Oc*V@z8eiT5$aQtU;ELyE!d!;fb`@k9ns-tBE-(U~n83Ll+wpVJ%d#-Q^6NGE974b_AVj*yt+y zj4XbQ6r#%+u>$+qLjge3NU^4Kad6ulUn`{5Zxp%mSp&Am4RYLC(D|>LBoP3>-uc&U zPCWi@af1ppW+@-n4Z8G$-b@vB(c4u!Ezd7bweW;rxM)*vh0Hx`IC47+UXU?#ysw~` z+N)J?iV5;|2-oSIrt_FOK3wqqla`=?;|t0qTUBcz;!1HPjNU9XQzd!~9_@f5C{gA$ z0@El|0yLu|;u)<+cl~qR%Ww@u3a#ao+^YuD)kSrlh@;xiOSD z*mq_uPCVn)848PuXnHf|*6||4S^Nl>0=*008`G4*q*XR&-RVUjG-Dxd$om>+vV#p! zqx!C}D8j}x%>}mKE!o8+ixbo63<;zvSh%N5P%_-v1tD?h*us9LjVuQVt21ghADpQ1 znG+3hOmIkyLd%&OY&lx&&p@Z2O_hO*-V$DXV)1Hl71b`^=Nr1O ziovRx5y6H{b~j6mxWz@8I+=K31^D1v&&yz?1kKdTFC#}B?Mil+ZSiL#9@w=}ZJFfr zZbd;32a>; z{x6W2S0NdeJ?~(hUbSLYAE$Eb@4VpA(N)1vB%vX2zgsvUx(+Qs%e=jRHUL}e<1_PO zFoNdFo8>kAmYMC&#|foTPPEk;n(wYAuYi|7o~pK)%B)A2pbSvQ`CYY(g1Gad-3?hV z9hzDWu;Gw?VD_OKwin1uWbKuW-4D)xmF%ogxM+};$2Y0(!4HX=F%VzVCBJO-yCnfH zA;wN>ALwz{6Z5ypSf5-s|5kARiPrskqS3w9rZGX~%C+AmI_g%1ZpblY7mYC~KMu$r z1JsN)eiq@01Oo(UmYZlsu1A|g<%Jsb=Jv(EtZ#l>R6@Sj1lBif8qD)VJzQ|&&yS<= zX&3H3NT+j$1;vw8FWI$``}i-4eL?ZJ*d=hLLhgs&Y~fYOJmZED?NZ}03H+eofuG5X z?r60%6$mYW0ozE%IqUQdRVR~>hj9-(CjR})p(gaFir8T+MsR=CtGVKa|011hrYeP% zK}B@rK8UfRtX>;?EU*aE|L%uzqPSG_@4p`s8YE~L$)Urprj*51U<8sok;|^eEjsg9 zj-kjYS23T&Svg{;GBE`-op>uD`D_N0r{M@v1TnnFgvd1?3wQr!xEgFwV70k!~cY)>{=q;sDq;%LYyx4-%SO2L1QM36CW<+<4|UUF zlr$mE?abB*oC4fr)iXLemX!APm&F*j^QyeSd1Y_6H&h0!8iuRX_{jj3IQ@e8?8KrH zIv}t8T~=3Jjz;Xi4HpKE|J-m1tA67;C6#XX8C?{WtQIYzr^DT#lQQk9K&Hj0>SC$Mf=(}nugUjn4G-rLHvU@_+ z6&T|;VyUzL@tpp%A@XN)6)$F)clz_(lEcpBwsI zys{TPEt`i{(0@U}in{seSVoWI;rc;5ONWOwo1X!k-{tzQOJqgdkkzd~V5Qig3WOVk z@wK+urhP&|UrNl#czhq&kNu6}J-eHn?AI5tEHc(3nS1t^qS69ei=+#WR&F2uYNplq zN|^cMkpCwpol}Bq_=&N_XVXX8-%pL{RhD&A*6(J0CA0g1iL35zUmnyd#*<0N-GQt;7X8+D%dzRxNV1uGIuDx`a*7woYeHEtVr3# zsF2}XVlW9ANDe@)h7HQpHMND=Rc^5cbL$BgnFoWOU{w(v+o_flTAeUEFym)z94uco z2e0xB?4~%e8GHE#C!NbhGKTv9M>?3$yLE+Q%HWbQ6S%3F*P5je(41%x ztqmf@?HG*XRjP?7?B0lJDs3@qCZzS&XHl4~-9m#Qz&%MjR}ZC#R2uhV-IOO4O4N}8 z@yk^Lf|kvVWcWmNR}4!B zVaijtR?}E1S<5OTl?t5S7DyzG%MO{w4BM2P;{wSSc3HII3zFxrjSd=sUH!+8m8J6~ zGQ(aJ*Y(g9S3A!f`9H||U=7m6_^KpcYq2UTe|e$hFe%LdH|}nIA2^o$%FtW6uQy&t z41^_esRY)owSXk)=!dOU@<^H*cKz!L6<-^?&GK%rl&aD~83Rb>rNimr)Cmwt^FKd$B-8e~Z2|CuEUdXI2 z=E;&~sd?{~Lm#-$^_Uf)Q+8~flTX5ksXU4%os}__(Ui(H-#_4_OG71ylpv^iwPv)&FxdOj6+~<-(dhZ1oR-Bei| zz?TgnqQdP376aL-;+JI~#)5G-_7kV0WHdtx7((znaX7JfF^~Md# zaqCpfwA%};5vNc!h!OQYDqT~Ex=)a@79jaj@D!gMupNXL?vGD+O}3EyKUM5h-sspV zm>RAhT06K9-5EfZ^cTcm@Iu~Y$qd*G?DD_Vv0HBTwa)jO^kg(3XG<&ev#Vo;dh-(7 zOI&n0`_|*Zl!!g0$yz}J&&V!tLf33GOS?`ol^{6sES8RiK4f=HS!BGuqB5T#u>L7X zl`fOB{P+BVAx!VOgZoK-g{^MWJ`1!q=eD=vAqPg9xMN^Bz z*2It{T=g z8~KY|yREsOGt}y_GxvG-9a7ZnMA0Xk?Yi@93nujY|JeVYG>_@Oyog-?Bz_p^;a6iX z>psHaP(iEE<2rQviS2=8bslO@8 z4X^)2zf-g*aprrP-jtY2C9p-{LedapA^^>*ua?Nd!s7d|gNNzwCstKoNL-7mfr^(; zL8tFs!Dhd~N3{v(9uo#s^}#RRxteY)p1ww?qH{wTkT0iy9xIu1e)|8JBj(8iBmZgt zw5GMGSsh`2Y4KL6v* z)$RZLP(LrCY+0&lU~-yPj1>A7k$r1w0G;_OCpjW0XwYV%VsnRl}^vo=?) zBwh%ohFwUb9J3z_9D|z5Dgm-P!X$2i<3ec#RD34!GsCei7|gPwj~Gg46lTRqy5YSM zv5@Yl8_8nlZXNPnjp)Y!J(W zL{sWrF*|Rs37D}_tS-(M3+jdEq4M=hsdsr+m8Ytuda8&~J<>tq6Ww$$iPR=9jIlzOi<+F|oHmTB=GdmCKtB7F-HvaiJfI zn_7v<&+@A(3V?zF3RH@cWAwz%p6|N+cz^}Im@7RLra2evnRTzs6=w5#Bw;uH#tcqv z$AfD4hQCwzAqSTG^5cHFyYJ-Kis1IPwsOd~h0tZ4c`bsR(USvho))R(28Ji%(93|) z+R^nyX(E_xREm73Zm}bR?um8J$2n|8EGOlaBlm@pWeL^p3;ZivTpkn!%t`6m)T2wQ zAnYdp=m(8yo>;P~;{8|hHet9{OgU-;IaTfRj#2cq={=ja;8dYK=n{bpF^Snj6xa0KPG{V?6HlZLpo79kw{&0g(g zCm#_kGVWyudo-gj4@z@E!^#EpC+djnABqKqS->Fm1cYT(PNxB8{VpeCc(-X+Ar3$O7Uu@I5d}9l;Rx3=7Ja>4L?FfO-;ow zC)dgW;IqLN?4DP#WU&0HbTZ1w+f^oZxE9f4kdx2-jTR!Y1;hg7PZg6&Gm-IgywD z;$l)JdMY(^z%^}Xp`bsfV>!umxrX&gP*T&d@bGLnC@Gu=vx28f=5P2nNzzGK6XlaX z@Puc6O!R4cR7`o ztK&;JBE6y9{Z21^i4ZLy*ll7jusKHp?6UDwQoxfMB^M6X$kbCAQXf!LsW7_U zdRzD9dpLXSWZQ6<>G3l(S2D98p|qqXb8pSoq zJl^3WXR+@t7jC2_-L5@|zIAbwTTQalMFO74R7-;txvUQa8Piq4M^}@c84Ev)46p1{ zFIWoCJB_-?+81n`H$`J!wYqUiyR86>Wm{_Qa;q0I!N9a)=`eK&F>%FqEwgaR4TWfD zI8&p`-YXNV*;j+GF5=l_J;c0=MsQpUJSo4|lm$3~C7U5rsi}C|N%(q%iII6fIoA8V z+OOQL;U|%q$|h}!bkdj!3mrlUE;b6n7|C*CHH(zy)(!nFN5uBRqy6@M%DKv!FX2uZ zf$;m)X_yelif@9^9GTN(eVAMg>z58vYS^tD&~PUVyBZN*Tc|4aqHsg3H8MP{H=*Iw zp_?hjUI1K80wxPU!Meh!u2|sPnCDz3p89Ow{Bf`#@QCCQtz&=f5~*J3f^ij$7g!s) zDgo##(oadi(<-IW6pS|8p@zT<{Jr1CF1pdg+W|P64RN_}qSTn2(v%b>eNN?q&%JQz zlC!VOq|qt#W0Im+x`qKfs&~kS$KJKK70^7bL_Zm+W@FlXYiPe(Dsl4H>qg5N7w$AI z!L$Z!b|FH`Ev@+s^cS5kNY)ZRS{=6(m`@;0v$=O?2#)qjy(J59hj6YmdtLErwXOz( z)2ZujwIF?LyQVIfuU1R@{Jg2)a6Dkk1;DE`75mwy{qE%DYnF0KK+aJ@0x_fCqdS!p zZYCDa!V{Ek?~LFnnK=Kx6AYlEHyBSDa7On^QyNj(M%+tS4$T*j-6|^0(}HH%e6!Sa z77P`x1MtUL=RyJ!?Q$iU@tmJ0F3`ZwMt$m()7D`tm`$b;Oth_L5}&}-gPu|WONNPE z!wKma5p^%n7*t`#^^}rMCal827!1ZGqN`igi{< z*%meqK39`~Q=gq>9p!632CW~2Yo8h1cdRL!(7z05sVTcxbd&qFw@_4xgPcMzH4cFK zlnSKb^l5p4CCmLe`_3{X2e<7eG{Em3ojw!%$lUT(QT=V^ z_s8aQuWra*UAq`1nOYg&f!ugM^n&!j1Kl;*8+5sF>UM`Y zFgwR9x4Q&zG^~uY(XmocH93kAXMaCw(rJH}!Me=V8g6Zd&9ZY&ztAuE!Es{2M_VCD zWGeW&sT7Wv+F*nbrT&y+5gg4#1`ahZoEw-5sBUaptXtYH>`5dV(-b@TSuou{tg&|J z^*1}`!D}+OMs3D~8%|65s7jX*1e(zj>qH#Tu)-vODhP2N%&$HXs$=djcgfQRBo9L? zDE?-xESG5@x@PuhC;;g{Vh@HKVQ6)G#<-1Y{OTMg6(%x8T49Xinq4){&o>71bQa{} zozl~EObZt|KSB{ulXqfZ^ha1R!_s_(nlXpE=vavRu1 z)`Z2XTAblr8rn>oEa90)m}m^ER9+muc@j|8_c}yds@-Gnu=rh)(@b0AvzhZVfvQEi@84C<_ZPpG zQFqNmxfpwGGU3?BxJhUMkcf7~Yq$3l6oHHW0ZG!|+Zz>sAa$2$&*(k&!hPF|4f_kl z@Y_P$yE`_o_+LpH`EW#0Zbq}LRZ zCa-^F@Y7Klj^3t>=v??d_f=;5+r^Le*PgXm@Ax}?zk2vR&>`=Qp5Q0X-yuN54rBEH zjtj=L-KJSs5IO^Dk=GTN*c+$0H_9 z5Uk1y!#q`7!}F`Yf!kpi)eJWSP5n0CiEl5ND#B%pQc|t1!LOh7z(4bquJdocusb*W zMv-m$_FUkRGi@o!-OD$7*yO<*ioU&@+Cc{w4=o?3%ck%(`viZz`AUnmL33hp2MN6S z;EApG@1g7(40BD*->+B{(3s_~?|hMDPPutUyy^E)+I*@{vva>uP~RX0{jG%5aE&Y1 zS!RC*2kZTE=;-ttMbPDyW+Oy{{7X&MeCkWoTPK2l*0-o?Cewm9mvr{rpSHqjWD?zf zBVHHr{FgIwzqie4^+6WR=Cx06zuiJ^NZTfN7FNA^#_)A!%Y1g{snt=xCew-PL@=#A z$)4Sn8059~A^v(sr(lG;D?opfB~&M0TtqH(jtlrJ@iPr3?r*9?NkOu?;pzll#5Z4J zZd)CJ*HZ74;9nr}`FR=qRonw69s1I|-;n?xoXFwSzcNu@z4t)3AtnBFgW6;PK!M9( z5T(nEk!Ube=M`is)G^nkM^Gv3Rs&kE?+qk@!~t}y+HU`3U3JAf)1>AcUYUlwS|lMl zOPV8`QVDnQko=xyE5n1+1s`Nd;f{LZ)6hsBxNmyC{LG(&z@Bk#uUl$sl`4jCv%_u{ zN~puydTxb~P57b>fNF50$*?URuZ9%gz(mIU73wCX{$D|LJ~6oSpGlc>ASr zw28p;u@i{3qSZ4Q_6OGYtGZty{mg5$XiBl(fQXkaxhD3`eFGV-odJPsN)qJKwr%Q}+S5z`cf5Wj%-s_c~k0+=v_QrZ8`*DzzOD|PqJ1)>aLLYd@tA;|H-^hq8a~# z3f*M+!yP%ttuzHqgZa~6>MKi?>XG^c>q!mzLQ!Qz&6T8eBa1;%M9-E0IPhAcS_u_j zern3(&&63_O4oEOGxgD@FZwkVrG(0szgn6@s(-xMMdlw1>rZ{uHlyDc-uan3FXnaZ zDPgjEhS^8lGE7v7ujwxZGfyU1Yf3yHGmM`SKWs0zL5Q=gY)6Rhxv54v>uWUIQJl(d z{P}tO_rCx5jjwaB#N;<{Pf!DLHrXr#LIkExka88+3l>Dr)}Dv0prBV#CEqE7sYB5` z5gp;sVTTOo#tQl*ZHE%2M^$H;u^-nhbldj8Jw&mE+pG?( z=_&y98w|^We0Qf(15+%JlAv7^FE$*l40J3NTi!P9dZk$-^ImpdB$}zGXm`pg^Os52 z4_U@)H^+)#H?4hEf1_w;VdJxPw`bUHvd@p1BW<{n=G0LdCAIYVM}XG&ZgGc0J(IY& zyEtHhpPIk-XusD)TPd5%_+)7w>u{ZUQe_~l22-qmy_#njEz0pI8ha?QRhY&Ha~jBC zz{g(db!#)@5W*O()U?b0YHBn1p+50(n!}^l2vTTq-WB_5RHWT*Dq1Wt*o9K7lUS}P zpf9U{7>^v*6U(fMmE&-5W2N8G&z0O4R9j7eXUIrQl=!kduGVUr?eyv@^B66Y1s8To zyyj;N)4R$R!X2rlOaKY@lAzOiVJ1}|N1hPhqn}MbbD~EiR$4}NBW?%PnA8Z$@hMLz zZI`??=XX>CYIZ#8J$zJ5WZi$!Fx8H~ znJ9g;A35B#Rx&;O>~S2FPZtS_XJv~#&>+ZjPh!>Dds%V3--fU+ z)|Ys7f#Xt(sVZL9OAsacE)%E+E6*A}HOS&b$DbIA zXSC8aZLYi>J}orH;_6vyW05~~^%MKg;eVo1X#^y*}=F1VG)?<)WNi+T*1R+K_sp+)jS zWZO#CzfJZ3qn&jS>Q$Vh>D`Ln^Av#`mGlkf`A z{@%v&!zG!t{e-XQT%v!Yp!{|F+-qqf?W)xzge_hrnvJbFE_%;ZmPoGKWp`(g`}Sy79Vcj%x~6D=jdpnY}+p znxLz1z`N|%s}!J2G^N53>j&D{w{RYBLwLn_z?3>qz8nG;0@fk(Ro%17@m1EvT6;Im zBs@xVDaqnLR7$p*4pw>0#2X4)DbX!lzdlu{@!0+b@Pmi%P?yfL1>n%E;<)RGh22$2 zl^1q0;2eLwI@}|TP;K1EIN!C1;~k^AAx%vjEWn-{oX|9!-Eq(>dnIHMRmSfuvd-r+ zj$yjwywU13dVRW*cW8tKFZzL$u;Z$q51qT7T!*b24QNsi1#=I$RF@AC$vB~3%2BeZ zePB#)?Db*%&Om_TTeDJmIB6Xlk_wH>6IWZgEI@|7NS$)W+YLNI_b}N{bQJ4anv*!Z zR%97Ohf@MjvXzP|oU~5%l9>TfAhO+I@&bHH69! z(mm#t9gb((aYc;BpB<^;5!$8&!^mB^vr8@wf?1pwdF5?~Ca>cbvwK2_JO^>{x1-Hu z+*tGbWEf~CuT@6bRQehfR&~0vQa+A%9VF1Rp&tsWEyQ^RV|2ID`UF&o@!XP*1)6Tx z*Ck#EOK-K5g|EQIy2hKN1`M5l@|J=oeBh-FRH0~V73t2XF$;w6hmVR)U{Wdnj= zr%J)JelWWW4OB?)Vae!K8ziEIegKJT!PzN{Fyo;h)k!is{jaAi*$m6i2s>D5yh3a; zJ&X=7as*)PzOCChQ3J4e!9}uAAt2*oqAa4r`nJ;RA3Vt?&yB&KtE1moiFw_w^QZOo zSE|u{k5BZJ|7kGet1kF=(CM(ldETj_A%&jg_D@uXY@6X3joHe3hR+0ZPk>2}wC}ihD%JrRLTaN=O{qzgiSh!??_Vv1I@L!Kby`(-7#`i1T!kB0Q<~Wh^ zdt}^95$~LvwW~~5KEd>&0R{R204NAv7;jDzWYP5w}(p2%T!rSa#%keyeq zyfUBos7K+&kDrlqzk+KDzT9Xvj}??}V;7vvT6o3ety1{;%Bv^IdTLbF%%0azhPoCZ z{o}qGtkCE6Jp6tNpLa&r_SvbS68v|_whDi%VOXWoPqVI{`@hH=h*!_i_rJH7etR=V1|`~E&XQI;=J`W#qGh=@R?OimyFA4i zgJNo%Zu=?e3|fOqPgqsg28Rn2qkojHwy5SCmkR}e37at-1AQ2Ph;dE!WlX42wm~CTG7H=JuNV6ex$k67dm) zViHz1k@FqU1Z-W2e!MP*1!;A|v=Vh8)0FA}Sah!lgkx(zwK8Dh*AcV^pUNLfikCW= z5?88qodsbyu-??KDMpKXY$LauT~gaqjwY)IWvmylgLq*jzG0JPTs*(lc2Yr?_kNG( zs;B)W!_5Q>rgS}9?Yc>*VOR@+(a076I^fmDpPgf|GvR1VrD>sfxM(ukv-XT;LTVb< zQL4w{F3+0wV#>6xM6Tfcls1cFj?j$OHkl7zY0T{3Sf9W(<%hy>ZN7txbN$iF{&M%f zGK=%Y9|wQ>*uE=d!*L#C^t@f~K>uAP!r#x@sO4F~CjOQD(D$Q`vM4bWv59-S)7PIO z(emYT{Z!X4w+-yL2!Q*JL=pF$rx!;|&ehcrp8VOIr)rwP@WKvy(tR90K)Ut_dMh`J z#QQ?_07<#`l6Ld+;t~}dte^>nP?<~Mffy5vBXf>AS*Fq%>#J8$mR@4+MO%!y+$tyd zrEBD4%nzB$iN_QF`kPGUe`F#WTtfy_KKp)VHMCXV<-2>=!Q$g^#6)1F2>3a))Pi|! zc(^MImC93!(Lq*n#_74)Y!=TAEvC}=6iVU*B>6l8Zxr#WZfkYDbgA}Q_#p{7@hDqi zZNa!+^ijwt^Ue=R$j?XR6aV_3oCtY1`Wwabcp&d)2cvfePn4SyhKo`ivz$^fr{R4y zDaqm1yc)t=4%ZmW9M8qgEhbv^C!_Q~N`&|)S^Qhj3FO893@HUoC-iYNdIY(xGnx6E z>I)F}Rvd(DV296-rg%mw=IKIjXqBeT!SMwMojh@^hP63CyroTc89|*y#eg zTX4?}orVA#aW+qLD-3Z#$@;C@5vcu)k^9BgX|wMV-Hn-84Oj@!mElQ^z|wNy*`GjJK(a zJl8-!uy;Y7MkyzyoSk>%xu1bpMh@*HKIw(Mm{2V2Ie)$Xh$1fDe&brb?tX+l3k(qf z3cuN^E&a-KV2$xgM4kTLouYdt$coPMP%bs61Uu&uw2%t6XBQl;N&i6bwZtUas?|V< zDk8H_y*g~fOTIcYz3tXHV~()99q-;B#F6jK99_tKe{D)59fvzddAFU7FE2JY4LwW^ z{jy8Qqo1!r0JXi^60ikIiwfFRXC9sscqx_lx<&zG)VQ#iCi}>h$dOlD zF@Vv7A1IR@$JTeqh>EXVH<0sFW>Bd|6^;>RXVd62TZp^F5l8?=&^})CIaW#8k&{1F z9r&#^Ze*h1z`uRad4nOQ1SUjfK=g<#?X17d-CiZQCx_7P#9rkyO;Ia^%=L6GFuD)q znIy$IizIE?g~&cQTjaIq3yU(3WaG3hG8}V9*s~CO;qm}Qla+CXB2ZCse2jyWtrEWZ zVV-76OI;icQmohCu;q@*uCv(X5 zWDGiKCs1V)#>gaDf9}9Fh)p-6ou__BBlnaHKXo~ zD<&h1?deN)4O*zPqdiH~8aU$$K_i-~R?QH%kBGACSUql8FqkLao(qC&cG5(r4tDKm zn~x`c@AJ!^>`j42q6Qqo5)}g@H;6Q)8)*AwuELR&fB*<0DF!@P6G|iGJ4OiIb*EZn z7W!U7-c-k$wbvpag7ZxE4^}Eoma%b>1v&_E(8_pm^(;6%QIJb)kuzYTt*2e?_Cq;; zI%5}-Ab=&&X{fjD5D{l;TrxJQ;#Up4mxJ^n^Fc(b#KaY6PKNXwFkA~YtUQXkhcb-J zs1{c=P&qDnAiBMluyPhOS0{O`Wsb~c++U@z@Pk4Ze{Z2l`Kxb<2~?WQad#seVYb4 zQ|$=2&u?E)bIOqGmLYYgT~wVm7dWIlp~_Ng=c`%yq_~vx+Y#!!zn;;kfVsElb0#R| z6C5%Q+eqo$_GoRY!E+`?T@XGh(|nU8(~cT(yH&#} zL2t(`X*n11tPEN}!9gDTS#k62ozPMGc2V_MGcTlcH%pqHU?+BVF(eOgEyV zry6lifx5fO$|1)x!U_#_WMvW|Md`0U)lW02=jw2q0P5THpn#nwLuADb@IXLpj$9#7 zhTbIQ++&C4iqhg{>vxCHSK+H3035Q378vWAo&IEl^FX&=O7Q-ysUAQ|6R=Q)Wp&l& zq>Fu1so*_Ga44w;Bz>nTu(IEuFc7pE)5!1Dn0&MYVBlkQFu=wIq($VwS~PQ1zklC} z=y97-wgzSrl4dHAJ>}PKCAk{h%>kk7bdUQPh0fVe?24Y^PgJJo`GjxNvBX|P<=ghtPn^SD>%RTZxm zK3aV7Vg6+`a6!?|l_=zAZ_@K_?xLSsiAVh%mmrb$8Wh_;#IR@^*yS~{S~puMhEjEo z!E4vQX=-R(N!pI1p8|pGbD=}$s(517%`)bc0-u&oK4`HFkV6ic8H-KuBvmWh8*pS) z-u@uwzRA@tReHnz!XPS$tzt62L0u5CCC6E*N_rqSl zbuTx*KN`)ItOohWNtcNSr`6>t(d8Ne+L;sRVRz}zbpi@Fne$IyRL>i_d<+(mW&#Hx zwYUqq0BlslnJW$N8b!1{IyW1v(#jp*2w7Q(v>#^JJq{I*6i)dfsAUchZE)|&lXh&< zT~3bnKI15`ApXYjlWJZVv^JD&nrL5eeN@0l!cq{3gOjgKg`VcjuWqrFbTo-2l|IEg zmB}@4;YqNj^yGf$Rsr_RG$OtF^LS^?tfU?EfRF01fSZO9iQuF1zF1`wXz#fH^-JR zs)Ht?aLI@0*Q4-^Zx<6H`r#c7Hy-p9J!1dpHNLdru;|81$4y&ioXF&{5Ov7VN+WFVmUs30We!r?GJcM zR_^zMPWEUSjNoe6SMfA4zLz#$GJiEHA+M3i6^^`aNR{;neU7A>_9vw^NUkPGA z{^td;iLk8mzQGIMd%vp9Gx1^I%0khcSXmNY0(b!=AP7ndQoGkdwr#%H-6&c9(iDq& zr{7)+>~#Y-kJ4W^l^9*rj#-cyVDTF(toq)mmbfb)Y>mN^8j8eRx#rY+ZtK4O(+~Y0 zcV=I3wWB?lA$%1s+~RezW~VxCQLxfkh#i=8?2I>X{~#isc$t1O4g?_SA8B+{OH}ia z@yfh%zK7hevHA7tgsafmv8WmW<_%c|F?!~b#dzTwbr9ZSi@#GgwE+%?`@LWp__JD^ zlAF9hFPiR5*&Z2{OLUk_aBTn=)>Lyk2@03oP>BI+Tf?GjCZj!+CF0_@pR_xWrdP`r zPap3Q3I+2%f82*X=xG286-d~w2Da_FVMc_|!Dnj)#*!~3-h4;sB-Ru?{0eBl;bwd$RHY7*&a!1%N5G5+odI@oj`>_j?}VT1J}TQra=~hLJvD z!a#zvUO&}K<2m5$g0V^h$L5hF_i#1YxE@Sq?ogoa*i($(ZO|^7GO)A}Wm3^-d%V&s zI?s((}YvkuNf4$$C?KqP4+?R{yL$5yg ze{E25lja$ZxhjPKFz|t4 zrkNL&(Ct+*fShG_2^s{)oxhN8)0A7F%q9MvN|*ljI_H6mf|mtp{X^S3?ox5jK~~+h ziO)CM6mj1~E%zewiwWJU5@R@-&n5`8ih6`VU-3gc@&!K=?tsc(F;HP=e@KYI7IeJ56A|kCs#459H`A9p%xmlwb*#l1)&- z0{ZAm-bM6uL$$xE5`&FYK!Yno08UCtBq9K4!aB_(00({NBmO_^y=PccUDqy(<*^{3 zpmYK0H7H$BKspIEp$P$0dVtWRccqumYiLpe1PDqA9Z`@D0!e70SLp}>3JC7JyS$I@ zKKt44^PTIv&e=cq`T=vTl{v>;vy9p9agVUvusyD!#k+oP@QzscHEqaVSOm|Dx>f>H zmQ&uXejU#w?fudI683!FGj-Uhs7?8)*ftz)a9<#aGGn-mKkA~WRzsEg;#;#%Bn_R{ zg<>o{zCm(HQWc0gFs&CZZmA=7>#5usINCNqiXDCO`xfy1pv#EF@E0AJqpByZ;xS6S zl&`G0x|Mmfn>~|o#zJ=C>k$6QzqYQwy!&mX`|X-*Z+A0^DmDL!aL;>dTYMAe8@q@@=7RHsHr)PTQOnA zqINrgx=`ezqfMUll~hq1r}S&pO4lV^Li+~B=GYnBZ&A4!5;8CDpiPIK|GX80o5l*b z+2QQ*P7MdC1Rhn{$!o9Mi7vAs+Oq2odO%2IFOQW;{84#bDn^RVGUgkju?MueKJ`mk zDjXUe&~QJy7}1`RDg4$)h(ns>9@3cRSiP2$lMqr!_Oi1I%RR>{H|6)pih}PD<1$9F z1<~nM@wlB|@QAsErm!JguOn%+9i?0VgJ=1p!TbWUR~g{4){_j9LqQDB4;4zE84a3G zz99}6sb{#kara0q%Ss$ji-%=7xb!5ToF4)~22&R%-mn8!x4y6@dn-3ex){(bz5!F! zYd&MZ3rFZF2ztQXKT`K{DK*E(IPY?f_*MG3Fm$KYY8mX7q4p(WquY}onF0b{qw85~Cs!E&MBqx3#4Aj?x~X55wkFE)7uva|6}l<4`uS6nqSs^OdMKa{7P~n-kYyYaaC{g#)s&u zS;jx}2`BAeIbN-f1@R39I|z1Zk-I$qGXhn~82zg|HzjM1RsAm+n~U1%gWMf9L$UE~N{*1t)Q*r+yA1mB;KlaC!>J}>! ziPNP&BW%>bPbhEjd{}3p$l-y zI00#4r9SN{0J36tQsNm?X#qk%sVdtuJL^qdjWYu`#5?cRJ0}uVo{y?jQ7xwir99iyPqTkBxt5f7n|h_c!(~! z1s?(U<+e{Ja?Qp#BLoR8xl84LQiA}tqQ3@B+88gBP_3f3&*oQxVG#F%DV^tD(gvWr3f zmJum&l83CZ|oYbA(QoAnxB5PObb&^i+(qW|hKpM@|-xUi@o=r0=hs9S7UiUOx zQ7F;&$*lFH>>Kdnlq}9I zH-{H1Wksxs`Zj~i6`4VIs2L8I__4?|$=VGmmda7AVX9K_&l5rZ85L2*!Z*{=Nx~zN zg^+7)uT}dlJ^Qz%$?)Up%UgK6(Pp=t=&c{MStO;8y%2p@UkzPxYM(U8>9HigSS`j( za(BM^cS_SrCU@eo+Xmv3Ul1ba=5UVOx}~hhyg2^#TFxx}OKq*0?JFP1#Q&De>kTh& z5lRUfZiUfX$NE{YcHfu_CwC3Hi-np&v6F=!$@dpubCNPPe)4Zc{;iHr>eJ%Ke@7-G z)xYie??`?6|5Rlo&jKBNq`AZFDlWs#H2z8-mZka!kC-blUSsWA*1F^r2k2%Ym1n53 zS={{el|0M#LY2n-+Xv10#EQ&x&YJ`afJrKk0))RuQ~wv)MLj+X)x0iH{1T}m$jEN? z-zVPqno{aNdUb0cusHtr+kl7z`E%`{!@Oy^M<+`p2N7(iY8!eLvak0I9dds@A`ck> zC~_T`c2DzIs?CnOr(it^zHZ!H_CF5rDY|c4c(f01e308chwRV?b9%$*KI;%@!y6`RYpGA;K!?*_BX6lJOkiDtMmhZ`<{Kkr}#=H8!vfhl`@lh ztcd%QGp`;eD4`ZD07PoWC!AX7)dp%|(bL>*K>OmALuV}BM7c-ZnmPyT_`pqOCvZD^z`hdV10I683(c|+SC8beQMv0i&S%UAkCWO( z`y7zyc6;J}>qH zi5H9PwIr-FSL~s9X(O2_*~?T1(JzlxQd~zHGdU(kp$^;zM$X--PAchiDy+qD{w`*h zIiF8`aKC8P6G(4j+R^T5$$Wi{_Q-Kxy&vHLiN^zn5tN+IbW~0BkdVF ztKlFBl4sx$NbPTtyXsDq`9!==J)d2u@NT7}C&I*Mpb^(r1NPIR8_9 z3yPnBp-Kx4Z?t=x1N%{ z;zQbpi}g`9SQKTUmtGS6Hf7T$9oImIVq2Ym37!dd68h-MTUUUTPzIbSHVeA$jkJMA zYNKs~jj7T65)PHG=7Oa~zfMujSDHVZ9qzQG6v2ri3Nh zHGo2CwOObFKV{1oZIa;w8PF?6_cc-RfXQ?}gVI`N>n2(iHLV7-w7L<-gQS6(xEw>w z(U^^|xL=vbVH>J|UiW$$4^U3YY}gJ=Lb zr1FS968y>T`R4qf_?%&wDu<*t{}sbj*Eg_Xfd#6Ll5JGx+|#H-01grWL>JA6`HHFgnp>BV7G6 z10{yH+XHX@-*|snE*g;92S;4pqr481bbwkwS;&BZVAA@Xhr_c_o}$tz2vJerPJ;ak zOboAIbk4!d@=sgUjX!J0e_W;cOK(Yaxo9CHC1JkD^@~-9ID1`D-Qqy+eYD!do8xP- z-L!)SdLl?tE%rv2kgAcSVvYYB7WdC%+J6B~{w+KumB&XqOurhj4d$1GffE)W<=_6n z2iX(81m3uAli0BSZX+^jo!u7&E%qy0mZfmlCHuKj^}t-TeSbHG_p#$M(;h6O(O%+_ z+j61tjEoajJ87Yz!$}SSdekEZxs#5#F06IOx{oit>ru0#I`c!DEFD|hQm{xX1kJJd z4lYX!jdq}YS_bPjR)Q(0L+wRPa74}TX}-?XOc4&0^!HzET8d45UGt(eUM-i|iAWjmSWo`>v_QsP{@wr3C#8U+k1G_DmvwBsF-Uj9n{0}v9 z|H4^^Fb}ZNX_pD7D$4~I6{8Nxgm zHQ39dN}*MDG=BijaF6brpdv0pD*3&tm5ejP)|CBH)}oVksX2YvuJ=F*ExSZ>S;2J4 z;)l5fDzR&@VkGQJH8<;qgqX12imbzvhvipI==YYH`yyiOj?I%Jc#UZ+q_leal0s}+ zushXAUtYcy_9UzEw$=$}wk}_oe6j96WoMN(XQyq1+vOWcA6hX_4>R<(8v%^@nSFZI zjIwy%r-~Xr+9mIRd{|Mof;Sg$)Q!fAu7B`wsGLg}faWo)+x)CwktqNfXKIQ0_Y53x za98U{`exGaO8Z>%@f0c;Ez3Bn%5@^z=bL6CE4OFZBX0Wiq^xiui&bWqXEj`z?fMfM zO?78#Brnx}E8{#VC9OFu4`Q+`+ZJgduoQm#M(i2O#P)`|TMRHA|W zzToq7r+eXDr}A&%Ut}fQqwBlmnKpxR?{VMBuCw|Lcu~pZSWje}hv}4?Hfi-fFb9X?CEny+)T( z^B(q=6v_ z(=yOKUVe|z4j&x2dO@8drVJ|C@1Hwr2Z{F(;1HHboZeJ4bGekd3$NpcPc>ls72$=> z`qdun2D9-k8?rYw3qC2*IPhu-DCW6OgKIH2$2vJ7{x6+mdwjCM@Km#Y#UHlNq$@A|0`jbPDr z1Jrdzi>r_0H6t`-iio#kwbnAX#A`HmM2}?~>CHy7>>zAgKpqrzxyK{x)7nnrDx|Io z5hmA#)q7Su(Ueg!yXifd+40a$^C3ojNx2@PtYNe(!6B!diwv*Jo*Xo7dU};*+t>%r zcF8)rQ*qv6qE1j`=h~fh$hwX!H!_8ur%OZBf-eWOy>dKa^Xns8`iX}Y;!vlUkcxo z4x-zR>;&-`I-QprW0x0+dJ^M;b;=!yT4hSE>365$f+mCY3B`yj$*v?zPrIOQ0L%)5 zvDeSi;WQAzlE}cEBG_9wUa2RuOCU9`bAdEA6ZU%}2t@s4*XxRMqQDFll-RV^ZI;bW z(Up2Zy$$^`iNGv5IOB3RS{1=xZx*_d1Ol-tY7SzVG*grdNPR|C-4%|?SZ|0Bf>YGN zoX0DXXWKd@P{S=1YGApxjDxDW_WOm}bgQM3iMrQfa%R^~`aTMNixwZX`cmr9nAR&? zps&hef7^-e{*_6Y#jye)Bd_;BeQfM^y~=ez6<-zowYm#?`CeM6pB?(ApDrA+G10pd zpAAUpv^=??L!uuz&jr`BzAFkQYj==9e=QfbLei?l23^w3Xsjx)$|+}s`z4- zcxa|W8A*#q*g2;)lL0iU)dXN3xAtDr)w~VYI^#{1?rz3g=yeV_r$?Q(;X}_0JxG(P z3o(r(hU$UBR+m|N7U3{ZT=DSMMoQOWH(VRQ=a{k-4&;OoQW6>W(Ioc`R-z#pZ7``1 z=Ud0Pxt}%+RKb}669*dzNW5-KscXpy|CuDZ)3>0HHRNhFX@V1Q1+$vDMSI#w&08gB zuEZ#yWs>>~rBpdw<|?31jP+dU$fRCw`IL4S$X^e2<$;-yP&G7>1F^BO5&WxLF~R}g zYE^{|s+?+Alf#~sb5ss+YJm?Q57aUXx3v~zYVV*`*X?X<(Hr6#H^kwOBI+c%Gf`QR z?dkDw2br*u6sf!E_;vBIj2B`3P6~X5`5>T6uicbkq$sCxIpo5u61|I=jzJ%^=J`bbAh*Bh@~1%8ryXi@eF z)d05cmSTNAj4#p%upfW@?0G0RcK3F;mBdV}pwHNRUuN&m5HBdLfSKq?s%F|JkYimA ze{4r&_-BFaq*^iN5TN}C3M7$P9+dus#qA&S=u0|%$#%nlNS$$vm&0DF8+3q!6G)g! zSTX0|B$*+GdZkQ<$VX<9cC^NKR0?22$&;!B z)9KHVQDlf`DLO?pEDj`vrgzy`DNVCax%%I#{&G~bTXVr7N|;-bQ*0h&qX`3`R?V%g z>So~oz`xRSBw1NmS@&33Zau71^(QJ_ecsF~HEWm}TJemDmJ*5L4;zpr2=~ou``KJkMFD(r>AFT9>A6OYYEH<+) znOe|CCGKDZ923$;(GTqbmr%j39*tN@{P*IaaVL+W$%&n~Y$pR7hm>BFByh~Um}wNC zJFQg8he6)5io2Byu1HZ;Rt6dVB6EY^_ct4zhQ+;;ZB}?x&Q1irw*obq`ncHDhG;;` zppHzY0i6x&k|d@j4sjCUr2i}26P^sGmBCcC%12&OPYvNnLn5_1JBFbh0Zjblq8-dGX7o;$LJMP0wxzh2hW6W7lRRev!R= zdGcSI!qt`Z?LU5i^yTbBj%?)h)PBE}>o_58=JKJ+O36B0pD(Suv(YSeLs zf#Ma#W@b57L~zDKFnVXlawqVScO{OY49Dmd-pH+KA$C)IfouFu>b>52CIF$Lr&rYh z=tRw=Ia+i0Z2f03e&eXdlCiY5*cRl+Uu-G<+2=6ce0ncWjCoz2APvaF2^A1uj$7-i zhI)?bYkS9ztjpzOmPmR5E$tp~EIUT(KXp_s{^Xmg38T*95}k@|2c-~!T*^?iNA+;C zY1rA7oNO8Lk-#6mMc-V}Stby}3oa=y#b?3lA9jb6npAor&=XfbfO-ktUAZF$-m^DSm00jM%l zsyT&3-@~7e5@JFZ?wo3EKtHnsN^z>b4pRgy=ZE6Yxb98yYyM44v@(!zeS{`DCv5ay1+R&F3SqXi;?xwfw+*67J z@!uP-3`5H-&boqrk-f<~-bA~H-@ueA1ytW+NT9&w;kVsL;IC@X(jV5x`{Ko&ms8mDGk%9HaSwaJW|h>G}l zCdl1?o_}a}5~gokEEts;6%{)oG9xmF#3f)S^BgKoNR#7{LyAnjV&N%R7MSvt>_RRj+$*r*orx~ zu^Eu#fE;O&NS;@ZyK=F9O39>HYGQ@1H>1{j0+=-dU#2m18Nut!)I&2o6Fdb}$SP>f@F z`5u$-;1BbSx{9q4*fLUJy=u22-eWXBiCc7|2vUOP!heFJl|vv>@g62ZZL`4ZX@KgZ zH|Y6tZa7+H%em2TB}$_(he zvp!R@wu7t;GwDuUKm8}Q4)o-VSDkErs8*)E#C6Da|yP~$O zRv+(qm>y`qM$LLXWQ}iq>W;EZ-Zh)-?HFN^EVROUT#!hd2uD%t}QDlDdU&p7v_(LT?>~Juxe|4X~ zw%ku({PN_7q_K227~dis+Scs;pcnXFhtDE!EY`>w&G% zuESKCg8_fye5Ib=GbMiKf}C2TD_n}M9e~k>twon+ppFs}0UVJD3-9lN>dsBD5K+xO^E@jg;aSg1+ZjzN*1s9*omR@S0wtf%13Hmvn9r9=Cu7Ye3F~6(17G% zo>uwfExTl-?_{2Usn4}i9^*uQ2oqcxwgy?|`bE~_Uo8FkL+i_n_thEdX+{?;dGA-G z2R2Ge*i+b-%@^MmF^A{=k;ygInuN2N>dJ+PiVd5-+qu~9d9K?3oyw|b&&hEA52gr> zsHbl_-9iA@qoIJ(XaT@`}7W_9qJ?t^}wQSSXE{_#8MPfmq}}T1Wlj%dz9hAj0YSCQ9Dl_MkWt2Nz4j&K5i-dyzLOO zS)Y3ktzs)WLvTnL7hldf1DI~$O&A7k$hWoaKCvj# zjdH=wi)(9yPZ#4-G<>jZ&sW2(R85LLrCVh4i&pnfF493ruArKIjTR015?Sxc7)+9j zcNdCoWeC-R6ZW#8U8j7YA%l%V31>Pn4z-NtWCv(dJUz!ymP!L4oj<`*rl?1P@munf z*med;0bdT3YdIACDXob{>Of3^Y(``j*KBCC#dhFHBCwsmUbW&Ycd>vQPO%Cr9w!0x?%Ek!87_RcNDy{59$x9^H_tm?47(Y10l4ZnbR3YRc@d)bfF@6PO(!8RmHKLiWTahn9Sk8DiH6wi;GV_dl9R&j3 zR>j(5bPkkkd-S~<#yAPmjNCm*%RN5)J;I;OtZTS;m`wTS+7usA>LxFbn51(ALw?sA zzhSU{$5K<`{t2;mm>iqJm_JoxH=&yrjx8Stm|!I@2U1gh#hFKL!a03xfZF8{4Fg7M z2XyAG-ggrvI7Z1ps+bG3o3TxgjYQw#;c?zn4gH0kF9psdC%(KwmuLfI{|42I{a>XJ zj4!!OA}zv_MIZj}_=HS?Reb+0 z5-FAxmA~^PUe9&z`@REe|9&K$@7SA^fi^R2HH@O1^dM7eINKZbdo})hZpHsT4qOm+m<#*z zt(h&ngO>WwF#UO6y-`VvfOVGm$*IOP2UNw9d2z8(q|2>X6i0tr`|Kry7aFKZuX|lBdZg5idZAlYL|y=M zKPEw?O7wY$U_uYE?xJaUID1hpvVveQH#Y?J0#aQ+w27KbSw)E`U-)%9;gVyv{-ZzZ+vXB3e zc)UOB@_h!9=G(A1+E<7-yP?Rz&RYJn=1cd~!YqlJ-T8|{Cd%<+;9}Z4)J(jcd--A^ z0zM9pc^$!_r1gr%^I zmLnTt*;k>SR$zDNKNHWwSrx z0GR$sL&q{O(MBO5Ex81DG|`3a!9Hlk*>e{GCvza1m62j-;zRN3a({=TpHAr9n(NCPW~bjGh-AJ zNZqr-!^V#6YM6`e_9_3^5p#Ioualo<23L*XD*c{ik_}^S-V_DUm>_W7IdzvGnGbf} z=&XhMs@!-38;u#S+R0L z96f0`q=ofJHl@_Q;g*@u+Y&M(Weg>`40VFNPbvML1eyWco8-=3Dsx=S9HO$Xw2#Hz zMBHeC-(O`anZ0-O-ESyElYpjH-h4E~i7jzEe@1*g0*l>fj%(5qMB$7>}sO%+QcT%Jvt=?;Y zkx^F%llo`qb|d1xV{1)7#|z6QUafL`$;fQ{?E+)V01Ex3$LqT-wQuU7aQhXoLva{M znKqMA>Gd*b@EO$zdp7gawJAe(h2#udDbEpcaWQ31eZrSKv}v+dJXnyud|R#K?CS*t zk)3h+D?JA+iQF%|Z+Y9qjC@V5LS#i{*cTV9b{bEtumO1YD1A3H@v#8mV?#2%m}fUlF2 zzHh^v6jNi)((c_J8(lL7m~_-BWwH|g2q>+7XpB4b0CGwIv>(C%`Zk5b@wGA^$P`@- z#al70>c(mUzMk)By30`1UMO)GGO?FDM@Z`e+NY5{_ z*WrDnWH0Jr&63@%5ZP{k2y4iRwb@(f{u}(F%XUr2_k>VEjx);I7nEk|~qElT-utO|y83nIDUKtXc3?9?KTS zlr<=ejlYor_Q#EaEzj=;tXEaLjFeNA3S@47k8+t(yWY+@gMQR>1Z`sYs@C4v%wX^? z+@Y$%ZX6JZ>q>Mzo3k$~XXQ$Lr}bb#SUjvepHNpMiCKj73IL=G`q7@Z$TD6;$9XsK znAS^L*TXz9a_-^OPEe5*DN!>y8D>$BUN@u|teeGnmxg(ik3?b4H*ioAOe!wYh2Qel zx2u;^3`}=_WRNVn{Nf~5r5sY!BcLLEJvwh5WKE%t|IrKnY3afg5HE=k_|l^Z+4h1# zUQEY|fYWN%3}9E)O8}zDKY>D+NYdn+q3?dA(?1fH7b0m%QGFe}{0|Y9AY@km>I~VGSadMKpHD=nYS_LVbi5&{E zp;q?bDUJi0wjFX#vpTuwB`5IuQPJyFy+;KWFD7C`E8JBpS6P{<|wLHmNH+JSAS#OOEi1t~sbddESvF2Z7EXxUJ z-Rr-|z&-m7QSY$DZ-MdSeqV=51Ehp{c`Ro8$^tBc0zTKAWgUhyH0(kHOkD8KJ2K*b zk)@{JRct$cpNKslDLdu0Gm+GzVb*UPH5giR13T~3;j%!n?>WT0dty&*)P7R9vedhp z8lr$rtyLy@pi+r4FvsmpYaHixiGC()#9q2x7f^p7-`}Xo6_AumbpqM1IimS_t9Y~{VrO!@uNp(i#z?lwLag3m#jHrE$vR;-d z$IEkB*W525f0&@dG6h-q6Ws3US_%v9s~>8-W#lHupYX$6RnBN+p4(4svVr#o-$zS` z8JWBE_PV8*2rD(YmHr}oMQ<0gcz@A1bDMK3sN{X75w4oAx>C$Qe^8HlhVdsSPE5&* z<6U-czpCGt`uJe7ORkRpXw84`y3i#x_Sp-uTdS?p!ZO)LJ$uj;Q|nfu|8~;(-Rh?M zYQmzWi%Ze&y|iR*Mx-CdHk+fT3)#h|$EaM&)-R+J#qs_f58*H}wj52z&4fz9U5+`aqyF^1yk@6HTBI+hS~snj zJIqz4N6y5GdEOha$ zClOhF1QpO+g&d-s9LDK<+s`k2o{}O?`qJ(_#h7|`RsiGCV?UWFYb;dDl{5O-G$ZDc zWz{2XIyAs7DV{v`U|s{y7*ig=V|E1F&oHJ*98-D=9Eui9i8Ob; zradBtt2Mn5fJaFBU7hrF#f>KyI>#$}j;Xivt5SYI6>K(Ac_cU%l{ce^2nh&375{@D zbSB0MT=()u{5};44ZYG1&$(%i>}=^Uf7-Z zV(f8P=^x~7p!H&YT-=aw892#+I!hy6-{_z3L;e{GE4^w2)mmE^-5$g2n1Riuh=F$? za>(&js(*&)-rT?xAGmCo+0U+w(k=y*EU0j8M!G3_%PaK|QJp52Jq2-&4jU^{n$AfB8!+iSl~t zKTZn&+x0K2L+MgtI-Xv_7p3}2A(aXvf`*rAf!B5lJq0O{TKsVfo|9sn+@7(`iaSbX z;zoukt6Sg0Ov^IMG6q@YaOWTVeN~eEIvjnD)YdwRjHOy6yYs99W9=M-z8A*wZ)~=E z%YBfO<>+l)sC#_MdGG=!lr_t2IaDpL%G58rwsfVkg*RmD($b%gS^oTOMa^v71N;AT zlIU1c+{w~74o82Fyzpm+Wz#yVyv4lL8icR8+f^;%tvf(32rNlh%oodw*j`6TAP3LQ zC9zSJo`+a$q2k;9xb;0Rl^kE!N!jfKvMYjGD1(Nq(Sp¬PzC+RZb;nepItFBP1x zRM6twg735DvxhnxQs7FXM6ZJNH=6aObXyy)Jet0>=x=86GRzv1DXA)44;IZhC5Hk~ z=mIef+qQ|dpaK44?!G~IG@=A90(?;;BUCKRNY#UEixPeM5U6Q!Uh47fJp7XN1(f@g zxT{oQ!z*jongS_bqpsrs;dzPg0nE5r`XHU}GR!(B&lq~N@{*txBMc=bp7?|GDNYP1 za3O&Zsqq8P_&T(x_x?uxQ~9+glDbTUzav%TOmgt9*4oZJV^ha9#BA!``)3*rlmT<9 z2lZid__NR9`1ACxOCA?6e@ChUc_7fO25M0Vo6>|lb^IK|9}})r&GU9qGmVNykD;xM z#s3#se(>ZI7zw`cZ>5Ufwp~u!6j$XR%c(u%KI++oF?TKRMikQoFls4$s4hjBPC;o zzjnq}+_rnbro0n`pBZ+%804aRe{)B_-?QyX>XSDL6%YYJ6H1`zAoH8dm3&s?8V=D# z@$g#qE0|Er?!(jd|<6QqrvJChy6KM@ACz2k(iVG@u3q2Qa?)l8$vDFB?Ey@ zHOdw_uqhqfGsieA|7o}qg#G@KrVTkwGFMyK%yUv7C!2iYp!;{kb|3r=l}CZ8=@!m0 z%eGz9PCHn5FYF@Ye9(;AlloyMKezT7fq(X0O8(8n6|AjEHRdF25Sjz_IF>R4Bb!IG zmOPNYcgG|rBX7RCX-IQ6f#G9lFBzEM1q1ZIj8 zXFULtAW=@dj|0iAy_UYxV|l6aG|zztTTMscfehbosKvgdC49xn-OcyQflEzhVzVR) zMz=RWr7KC??AVeZSFB8p0uv-~D(l9Qdr!m@>zLQ0`E$t&9x{D~+bhp1@z=5gur*Zf zYPL)!0R-yWbP7WI;Gp_&(N6H$A7w_R?xJvmCIXYsV;ik(G@nC|mIU6r-%R5EsoJNV z{-@{_qep2ey82%(LgFm_sT+%%vrg7qON5ad(rpg?faZGkC5_OEY=^+kTCnkYYK@N( zr-JotEG$%X{q}&IDoir1V6-i=R`hN?OSFE4nx~2WH0-%yZr5>Y|0>F|yG+e9O8Ca` zGcY9oA$U;KaGacYC$wvZQ#=j1q0CmI-*iUapv^*qz4%6RP5Z7fpVvz>_i^_jEqA%O zuOeRyXVL~g>Q#)8e1pC$f#SgAWgS#W_M(?@XsBCkbxjG1o_7h#siZ`wr2ur0A2vEI{cJKNL}CuK>%xs=2M@Lp(sT$dRE9a?E5BuJH{{&-MfTzuHRXpuuxZ+oqzcI^mPHHRkL0Z^%4(?; z3zb3m;Fm`^G?de4m2hKyR-Fs>+GV2C&rUuS<<2PB+e7MdqV^{(e*EYfs&=EIElYde zM6cTjD5I4g+q`294E-6W=VXtBQGBK7<~d>E%2AIuaOPJyv3sE3ygG!Lk!a>gbbFF% zUQf-~hm}o_(rMQWMP^Dt%)bQiiva>T2yW*DsGC?E&*$!^eaX67@aV!juqZI%QUsz~Xse#5*7X?~tOxCv{hUmSH(E+|&4%%;l91p#V@7-5#H$;!C7w{D z8LW1T&ro!e4E2wbSw=Q~Mn=|fC1H7~sxGPaJEtap)aXt`*j;Sk z+vBV^i*h!jF^ORELnOz2bVo1EL}8BQM0k z^8}PMkEvB@eS=hvYd0OXh+t*obU!7&TJ*xZV6wkvVkZBOaJc{HtKR=miiG{x*gV$3 zH1EJ-|T zSHeBBYToHlbn0Z8&9`y|$0DH1v`Yd3NGX}J%eX#)tlcg>V9)K!BH*sP zF54?BU_LZ@8jdqfrxP6dB#IT4R7vURA!%0%gig( zhVus8l{3GUONboAuMpTDQ4$$hhx_VnxZ2*aaugak+FN;Ao7WBSC2Rt)(x^%?dp&C1 zC_!;aD{PXn@CO7CoEvdAVR%>{dokr5F;x1&O{?!ThqW>^X-cuEW*^}&0u@=Kr2D6-8T|P zH&knc$B@)^auSrty}QKD75C134b*h{m&*n*ud~33Ol5X(Kv9+jgigEL5r?}rj7v&& z+fXaLIpjg4pDD3!jHH^9$r_UbV&Og>rHsX^2(46bszHndrN|$3*CS7(T=%j+|yD_M5@=g6V%mSQw!pE4XEMs4MruqpnRmS38YJ!=)&0nxs@a ze^fqVNoIk<+RpL2T31?Fle++i^OV#}6mHfA!jkBQCweM?xWAs=Dr#z*Lk@6lo57F=Va43Yx#wT!e5%Pcn|O=q``GioTZ zx>*eCl6noEN)l}hD@rx1;UD8-(y~tqP-Xpt7+A(u8f0w4zb(z*Y6ye7I*RL@2kkI%vwLYaqp{cz`Gd!mbEUQg0~5 zuN@86Rxs9V7kcO&A&LoGH#}(rg|Y+eWaVgz2csisuuY`-WqqbbC08wAnZI*?vpA>p zXy|F|*A_(Hb7ExiMD>eGn)L*$rrxg(FGHX00!`V`8e$LxNDQ30>r9==LjKs3Hbz>QkcUhA=m2p> zMbGC5|Ci*#w`l*| zS^E!-2LHeQm|CNRBsK)>FIe+QQ^TPk2RAw@K(N@+l*7GPW%vl*j z$g2RCCd~Jj3f!QQ2cx@!iP-@{EycrOAnwd$4cS{(Dt!{?BHKFqQqis41fNLB-zC#j z{`Tf!H&(u9R^T8VyvlAs_Sdf%9J)|$;B9*pZp7zopZC1yJMZ`7yUtvf zmlN)}@0nS%)|#1BDdR1iow{F_%RZS?w?^xZvWc! z$!7*f#pt6Nouelj;$6sh)?MwhZSm%=i9%kyWD$IUHgIb4>uh{ZI5|mta83Z>{JdGd zM7VeATaWTHYN(CP>kn#w1z(z9gZ|$3G8uOvvNNW7d+Dg|;k5`k1X^apBO0z&yBI5#czU0H|As z+V7#ygNZraw1Aq1nhq%WF5n9rzF3>G6v@k&Ph1o=b3*QvQtegUL^$18q4 zx2i12wpb|YUJ*urP)Q0SGP9hkpjMbX_c*LC2(^&x(%r-HKo4~1jtSZA`mfPZa`#wu zooa}2wnu-z8BXiT`#$Vyl2NjRnjvINU^y|_k;EW)udQ8aHD~!5?Z}0X1)n)y+a|3j zH*+4s{cb_g>v=t`{^)%@@acDLZq;c!{&vCck4bN@iCH|M`0TR~YU;-0V&XC={(LCc7<}H*+GRml<&GI&!yF z;#U|`8aaYn29U~NOuKXLsjhZfaRZ_{WM+ZNTgB4w{b=oM?a!rCo+#&UHM^~^Zym!& zqfEY(+CSGqatG5(zzw$Bqw3gA19rYM&R;#!N=}dnod2!C>1Fu&5f3rX`lV7zKAJ8nM2eurzPvIn!2 zuGHdiA;d5|5cm5SVaL^toyns*PkvTzlU7i@pQ_q7xJ1|V3m^q8De}}j7kpVjsbb|a z6K9Xb;JR*kif}PlChck_YswN@>Cl6_es04$($hrob;T3L?#@l*F|3@pwwOhUJ6J&% z6Z64eM3B9gnDoa;BkV!b9YglTUrcSnb(H=k$yp-Z*N){Yw}U6Esdc~yIfv@Bo0f)E z0?5Lc3?LbS%5YCCjOaDt`8v8gAD^%cE55~=b$X!y(Bxzy^nnHPLZ20WVist8C)#*! z#MY4g+Ali8J)|}N<)c%CH$CK!b$E-aYJxphMW~=H>!qK3VP4oTgu^C#%NYsK01h!cR{-}b#VG7IC_BF{!!F1$>b zPA`lmmhxVXfb7N2*>P~<7skxf!#~IE*Emg4!>R0Ne5ZiINzJX2j(RvLAWi8|F-hBeRdWROe911b+FB;Gq##+2vXSB2cyv8~r0Khr!PVH;< zwK*({A&$23(kA7(D3Zc^DC4w#0YDFXGuQ|~9K&gaLbNNPW*M+Qlz)-Bn;#4(63*H| zrXh2ae%5hCA;QF)Rf*7^IB)DYXNYyPR_sN6j!-C($+6&ppH^M>86%q)rQ(cLh*Ilg zP9w1Xr7MLdb`?(jN=I0lq<9rptQA4<(+_ePll?SELZfeYbrZ1(Cyi>OXC09D#$fNS z_;Iw=%PE?7mp_+2t9R0EB16J*U3Dto@gvK+rN#9EP#{V+>fWdTL}T!LmXlAQSQWoVk)yhrsBdz z42jD*31j~SuV7)rH$ ze8D;XSHO+`&8O7m=09DU?_IZO`1Y(S)OCYyN(6qSUxUuGl>XGUSG;bYks{M_Hlo-rLFDO z35gy*9Kj3a&d;t(_1N4O&V&z+-)_+z#vmrQf}xu_;ZDvYJ&KOb5)&7=h>6PM|WEm$=w)bDT7;i3RCys zI)@&7TTDBTkK_5?{4C$j2~mALT}!V8=~Dk^l%rR{(yy6{F-532*%J4D*tts2Wy5}E zQTdEN5ITa~mfeu)M0|<(AnzO#$j4qe$la0!{UjzV0VX5Ly@8%&`J(tis8`_VJC>Qz zsE{RZ)hE(^R_)TV>5RUXz8qZZrC*=>Obv;lT&H%2RV!Ef)1s}AP%XUF?Zvdb>ZNi4W|SF*Zcb`< zXGhj`7UvQ_wq1zFhOwR>9I}2yP43h0{-F~Fv&k~VN&QJf@6i!FCpKXcz3ZIAmJ*t* z!vL+ad*l74Z3G<_Y_#46gJnq|WBvn&dNb|ZlZ;08#V-bzjm)TXGPtW@~HieE+VKCUa= z%G4ix;c+e<%t{Sb7y8eoOT0A<=KuJ zI9Xr$YUZap(w4e@pSi8j+nmjZR~u_;DP zh`Gqt3hU!#MszhvBKl=V5@zpA>dh5F9sP_x4YaUxEkq+pqI{Hbo-oOAd1fqSuCT!n zXF!5(fKv9l-Rq~NlH^(P=8fa^bbArhoGXr!3%wuRWnMG8si|I{-C%t&-uUT_AoYp&X?jDbEy& z79d=O=+|!!Ipavv)P1vkg7w(v z>**_myZ>u{^%hVkbd~$>6^$c4Mz)(0GFN=kZWP!ovJ-fq;;Jfo{Jd*_SpA0IS9A@! zuAim6?R3AnaGcrd6<9FNcQG3}q{2MJzH8PQVVsB3vZ@v}=Cxo9_ik(an4dXrvHTg# zDmrfD-KpT}Rrat_G$kEq^lWTO{?35T$b&n+T@`-woXmbR?R7WZ$5~ycZSr`@IGZcf z2t5{3N>W{zw&2&vQe#*Ek;e!niwMmbiAkCPy^z)sIx*|<##AgiyBuC#>{$Pm?Jk={ z@56A!82ChgAF4=IM@p7d?_%++*H4S0qX8wy4+%`=JDh2969d^di)d!_T!vIIhI45d zPE(I#lIO&qkS1J1Cv*yvG_)M;UWV)5LMir%qr$oEKCqK+jHNV{WRzN|OFVm4+q0_7 z_aww4RCZI!3IpQp>orY}*Ykj^a@YCIp&Fgcsw1m_Z(!sixv{lmT@7^!A}CI)Kf`7s z00Jqq^y6I%a=hIVJEQDyf1>7SYb{lG48=!tc7=mA+OVFO62`H(9mvGU1fYV`azL9R z%NCE(Z`kN^g1~G1N14sBe!ez4LgJ=u(x8NX;WV$T01*jOtaxo1faM7_TVto5wwZMe z->uZ+glO?4c_cNENmHMZez_|-Fyt&5^+!%FU!)(%34WP+*iG7TBdPdK@JX@Z?srwT zht7Q(N`WzdQuJ2$mJkBVCzR_BvWYQTyzL8nR$&MvlhKU({DnmEqnUEquP*dD#Tck{ zUyw@p+7M(p!c2Dx+nTwJHn$42(!IuQk0@J8@}n090urC1+jC&*f^qNWLsI;^DZ!aY zb=XY)Aes`!9k0;sNw?_IHWl0=JMPuU4-hhr=sxB;F?jE_ZFC7KnZ5wt;5V0<7-a1B z*{%0kpF42O8`_s&YI$X2)~$TD!=>(3ZG{Yz=>HLG%3^jXif#- zLv11P&1T-CKM-ZI>_+vuywYFw=7tH|($^4>&s7l=02YEp(m^^O!*KxDC2@Tdt6mFn#lElMzOR3(eLU z>B16h+l84o(nW1Yg+8purF1G-&ICzK+(C_T72_({awMhRyF8 z19bUle#@Y`Nxc)usFEJWqHkTa<^cbx96cNTR@|^FbGokFT!qM7`0!jzK%Hrl~;Ru?H8<&tgDU%VccdD)Gp6I4tM+OQ&T~>=vX8xX!=hDZw2ujPI{}V@M69{r z^eZ3EM>ho=Bb*{X9i>6eDEz)S(NMOV$0xU?T~cgL0TdX6BJ-Ok%)+;*j!j>*b3V42 zZtS25b(@HCjieywGOtDny${S5JBis_QW6HA=pVJF**3S9XNALCb6#RWt-g*%g`|K4 ziuIfIcc+}@<(3n%c>&@npD(o`DO4{Ow|I-;n2h~h>uAe_b$7k1=yRs2$I(Fl6Q7^w zm@XFt>u#pwL!>9jT|4BHi?Qj3kF;Kmof=wspVt_|J;z@sq8>V?myxT(pOiv4MN(BG zx44Q7Zg5~$#wg5@jk?)N-DS*NS9*huFD#oPl~Q(tC-?bFdByKk5XJqtny#I@5U^t?!Zw)2)rjQz*!XL?!tc2;h z_1^K-!$JBEm^5VEQHjVO3x%F=xgpe*kSwJidNkTUyY1Hf?XOwy2={{Qa>)zhGST-_ zU8w1ttgK&?X+Pz6ij^i$fe0A}cZq!ZGZ8!Mbo=K9WyV%7_u|S2hg?#)2yDhWfLg1$ z z&?N#&I}9pyB-AENg%aFgptH(yCHD{^Wr9 zOA+Hfe*9J)zLWpEPrtrvKkaVYIg&Zw7vIDV6LdA>7YM zpo~Zx!0|dU8nWKi)iWxND`VCd;Q_zv@n$+be`j8wL6BW*sF);G{V*J{E9lUA~na>XI( z+jo)gp`V9@aa6?w^3c!2BD??eOJ@OFF{PwBCu`od?{>F78o2UB?c~9mTPT3p!m>*3Nw-&}~SXLs4WwLB5 zSAYMNNcg`bN2b1{Dbbx-9KA49G-|F-4lskWye()$lO24SUre22{zx7pZ05!x3j)p< z!IA+i)rpDQF+Z;u{eyOv4WrjP8Gx}fP~}Hff6@$|p+-B&GR2J5oe>dRnG-VRuxLdS zmtkG0GBOHCMxsi<$?LRc0~0-1ZbLBxORPQaw`&O2ppo~QiE;3vJWmT>O6+rQv&E|# zk-7n&FE^!XJB4CRW^{mtI$4*x5)kR>1txQS0iyeg*1OuEz#tL}IC&r{ME2H$C9_5a zVLjJZRd4z`gY;(SREz_+6JtoN?+RL<8f;!}hQ9(I5jh0^!*bW}D$E~Z*gp=n`mT&V z^z*;;61y1>QEKQ<8kj-xus^n4w=LS&ny4WlD=`A3CeMVqv6wpI)Z&BnwBR5WU)hY9 zNPfqpJos^EJNd~=l}i^I*9xufJkr+fU{+(wyRE4hhF-|sV1(CL%(vD9ZP4oIHrsK7`u(eLs^Y>r&f}Y=PdX zs;hw?TUN;)yr168+sRRCyA*5{~s$N&#gv5_5HJmLTvUe4+#;UD+rfkmY5^2;}M+&{0c zv@k;}q`y6!*Uzsb?n||UW1?OSuf8pL@+Et0D*VcC-z{zqiU3M)2uGwtdDr=enWXAt zO^O27Dt&mBF?KQgp4=wxczde%C2dyhcI;%m-c%_3#W2#?|t5)Tc-k zdwoaERcNa%!@M1c)m$O`(1^aP8wmwmceRnhVm8(@f?c;k;4iyU-Fk~i`f&E1ShFSX zj#RlU#SUG*A6AH4sgK5Qb)(g#nB1NcCNF?ucZ7|Yz5r;0(s+V z=8LSNv|X9Iotd9sA@$^kcTO8IBJ#0BpnThy3ZrA^3NN1pTln1%KZdVTWyf_+w%@0WZerUo`g)~W^(Vz=d|2MXwq8tI46mn-*9nu|{(=0?Z zHcc*cadd=yPnLW8ywlq4-$bumGAj*r=|_RYN_)W7hEMvE`h|hOoE!#fp+==Y{e}kV z)>V}S%2z{~-@zOx>(2DB;rKu*JeA3bvBnm6{oA5zBZuf7$HbWSZBbbPH+MeRcC0{xFyd+Iv_6`H!8E;NBw^+z3E7cd9w6$Jf$sLoLBr`bBm zWi{#J@cyK!b@231-J;yJE3Ru5c`@uAl1~nI$oDD8Ne?kwTE!^m-n=d|pyz72wg)pt zB*GvNwX-c`XoAiWD#gfkpyqITK4GVbE_q+mfib=EB>o&)9Y$(+U}{l&dDxJM2KVrQ zN&cgPgo}@_B+mcV`~jCjrGP_w^}n=4-mYJ&8tXWA4eY{;1Kjecmhj(xbZ*G<)c>+g zI(*xp=5DC`#`NiZ``7tTv6bIhvrN&mxuj^dnC9g~BLTvzS}!L>F4r3x33Y=eFd%VK z%|P!yFZ**4^HgVe?_h8t#rr{3d40gOD&Pekl&db-)$Hy!$dllP@zi*9)aHEC;&MG% z=~nSKYt{Rxb?q0R@~mnCO7OUix#p~C%`*9}xV9x&RNnh4PYFNhJ7Y3P%QD5?SWL!@G-Or%G>2pxNuNwx z5M^>Epe0t=azckkM=k{|Jkin?3#f+~@qd%iE7(G-_0(NH{oW6Ct7I0_-K|GZ=hrKE z?u(c#gz5ttJ!2~9NJ7Zxmv&9`S0`v+87|w`eD?gj=Ru-R5p764myOvs6049y7cH`1=eyY`DiUfHG*x)X>5)?<#$S8)dTA)9aeOh z=hzO16drIYOG3sNhT@hnV_FLk7~D|6p2+j-d38o>Ine`VdCpHVl%28mdd2O597$b_ z0YN>SnONHFFDlIe#|&5kXn{7oz?%^f5ml!G4}hy-i6X710y>USCa<>n&)4{Z;!h`g zn+bK7=_RZ#<_GIt79LKTPIt$)^l4++(piQ2y}`?OUv^teB2TOpwT5XoWyut1!Ga25 zoZWy2`BIO~Vr$+Ml<6?mFwW1NDk(Tw;w-hI`WFy>z~%QbR%PR&d&i&T+k|`B$iwGh z10$zV`c5V*BNq%lGZXLVbw>;k)?R5$FS zd5`F7+UFg?+~%lh2WWyK{j@5Q8%Et36STq#9s~#^nsMtPB}cDpL%T;WUp&B~eIp}* z<5TYFjBcl9os9Pe^1OJ!$Kkk;gj_!LIAOt29zn`_;<*>!ZU=3W>b-(Cq{1_png=Dh zCHKRL>)J*4hYV@hyEX7o_eIlqAr?V`X$kmeJN09@*-s|3y@I=`GjeZnk>JCZDXK@g zp|_4^j+VV|TVb-<+ewAe%sA4s+?%$`SQ5a}gGUh4z(7ZaDqb5Ov!f1V9hqS*k2lw9{lGyX(K4UZ_0!C94(Vzf2 zP}-$CdM^CwSRH`BCXP;XYf07oX@3eDg@~qOCit0c6cobEbJg=3p)-iaNv}MhS%vXU zV?8t$c*m=hPsbwzp>xf?BYh*>NVm&5O!QzpKd`etlK4%g&|(BVZR=j<%A)K$T7@MQ zIz|#dY$TzE=!Baq^%+v8%h8Oaxr?u>h-j7)FT?efZ&%t<9$6RfsPG2gk9m>{yxZpr zvRYS~CXxUmGHCL%UUiA7OE3rDzD(wWDI3jUEQ!_pPEC0?@z>+#!siK8!5^H*pnm0P zz#!0?SGICB+6%Uhbv2GRk(NB&$YEf9c{Np-cjrYh$Ndlp*fHlkx+O}_#GxgsF)N;* z65PssiR@nWhR0<>N_-u3BNekA$bIb8$hd+I^}m$AC2Aeltb5((QIm2awMkPdIfWO>)FMlIKPh?=ET zno@j3vcY&&LpE>q*7{sN{!I_+b}X3WR?I7u}|v%N#j%ryur?r^>r>oI|Fye)}>3>=08qY<)JbaJr(b{ z6ej<+Sz+7_DV^c(4k$n+f|K@LwQsZ;G32x|C9fqUulu)!IRbP3BDpYAi?$hFk%(3b ztCUUphmQ|`ZR%ug3H~)PzNG4WPY{y16J9^t&h1QOb44A=^S;^vZB$#3LNbV-0E!#u zNvKPxOXeSJH?h4!L49^TGQAB~WFKWAOqi|wTSM>9s|)XE$N&8v=V~EWpr1SIT3U81 z865lDar%2$17jP30=W)80?5>l8sM@yNm`90&ZH5T5O1z^%3iwcZ*YUx%OBZl26!I6 zgb%+rY@SHS*R5cBPn%_gNU?9VG!oZX*hVX)j>!sDB-S+wy8>^L<$~m0T(y>-Tc@DC zLp(>3W5$^oueGwgw4Dl0KP7DLV1}EDiL)9|BetXueBx&#g0KaC8=-IiZfA`)F0U!W zc|qlmR!;G~B;w;=be+N3&J=4zln3h#W&%gCoxmV^cc2QxIhR1THVV>Cu@vr#4dt<`Cb+)KbQ?#Xb5Wn*3g z!j-iVozD{T7HLS;X&&*UVaE-e5zi_80_;W(fWmw6{$IO}5q)~OQy6`!z@~C=qD?7J zyY-#T6HCx#V_N!mpdjQx0w$?cN;X-*`Uz|7sO4(?YZ)bQr$Qlfka3OVCtY0&jg)-P z6N7lScrS=X-h@$NIxNt?$;2Rr{rpVj!HWMlU@YimNY`3vJD!ohul}ePY@ty21>Av4tD;6D>$2>}Glb2P& zC^>dodKPfC*!YpK#~fi=TVfF(CI(GtOEAi`xeiK+2TXzMFS5zCi20aK+#c%rN^-VH zjb1JwI~(Kt1clcH=#(_T3d!od=L9*q2j`ZH zf|f`Fq;4{MmCe*LNu^9sRBBu;p$g8UV{A5!rlLb)Vrq*NU#rQk>C{u`gJ4G=8>ZLN zlIN}>-_fwI{{bx5|0H(1LGa`5bl2%Zq+8azTTS;3);F8d4t%NpaXdYI?V7;>tZsc_ zkrqte4}Z7#+LXq1l(p(Z?|Yf47q{MM)g=aHTT{cd>g^woAbcCLP2Tf@mZ(NnFo|C~ zzI;*5bvyi^7E^h40{nE%(`jiigyHoWgI9steHmYJ4r+N4iUN%kTXHLhoIR?#rO`tz z-*)64%96Rl0MD_}959;PHar$rZY`A=s1%b>d^Tcg@Lr@+w$W{kT@y|ovPJW*dcmh0 z=%>p!_s-iP6b@>-g%-o>(Wx$J;X<<~;E@$Js*Zcd-pG>4qe)&mBEhK-y!(kWhD3aF zu;6pvCuN$H5I$S767p-TTu{3AsPFQ?Gy+pBz>ez_>cLi|G}E8mjW-nHiEGRf zz<}Z>7CB3cjXYwops@~VOW6r=%xVu?RF=|0GSqEKwZMbYTiY~r8XZt=@`0X~qetpUcX;;A~mERhkzUn5)$V=-j zpc8Nf<@6*8qTy;eoFuLrQKh@XD>*b7GbZ_J10#J$DLINI`|(bEj_hakgPKMFATb0QV=`9yV?WQ5_kNJa@ER9J$stc>k4y9!HAiMzGJ9ZuvD9nSQOQjcs5+AEhc>M}yjvc~TCtKdiGC)L1PRNMgv;eep4G@OVIx6Q91|09@JMbz@ zj#2J^L$6Idd5+iSw&)(ZyuUSOL=h9}QF}<;Ri9Bzdsc>CVu3M~6q08cZrp5oMFT8y zAKY2)?@uh@21EL=Y7J15uxaGZsZ#qYMh$+RR*QghO3O;g>S5Ky4emN(+(Klyk4EF# zQN&tSu1}8@tj@9});_;M3#T87V#iB>{61i$J3Rarb($-wGY{pU0zvw z4*5I=r?7mLsPZrohLy~LPp3;Ug?D3E1w;VlGZ_g8MZw@QfeGGmpnuTRa$|XS{2tdfC*Be9)Ov*`8<-5gX9Dx5A5=8G^zOfCC9X?uvmn*FVOp~ zyw{|do&w(X^NX_3&*B^umMz9e4PYOMm#?qp29pkxK{Q@y1W@B85(wY>poHBvb*kT93rNpTRuQJcl#OTYfhyQ?m>;Y#OCY@KC?Z{LVoL$ty_ z8(=u#$#M9^^0qEADH|$G^)bAWhK9$RKbuT}^RBghGX8*>_>e|swJ2GM6A2d32$^si z;%UtkKFheHHmosPa5vr{B1q2XEo|}C$RCOuU0sS5mlmydIs1g=?p^h`IFh54Wt7&1 zyqQe9z~MRSBy`7j?m@9CgC!5(70b*n!l^GLGNuP2eU#zO{vhGYgy0%NYW7E+kF@U> z2#P9qg3%l)8aeipjSnLsBPa{}<(b`m5}4*+Y}fz$tlWXc*TPhlw;u%D=X0lSM~4-s zR4XHPeS=-O5$aV-SpAsPXPDzn2KcBCTAJ<2s$l&po}Wjl389J2=J8Hv{nC=xvTx8B zr9Ei*rAdlX4JF}3GN-<(D^1#q3tp8_9Qhdbh2EQLvQ2k#>r~_7LEBdcwWbVs(Tu%s zUX%!27-R2~V`LceHHUPP4p5(|$i+`kyctJaK>tb8*09FTdwRgkbnFOzqYH0iwwejn z1SSLHsHVwb#seS|7XU!Zg>PCG7^!hnS|6jKUi2eGfwP|ygWeG&GG7Q zl+jift7Ks#SLVWis6 zhO5#lR-7X*g6U^VxBWqFLs4+G=v&ngP%%0GC0Z&{oMS^L3MSMomT3eKLvt*P&oj7} zqnV9A69(L^Up*~mr@N6(rlg#7H`Nit2UKKO06#A? z6r_K{qIsw5xOMI2GYyPg4enJ$6i`#PvzlO;WrPtgK)8#XIS2I9ni;_KO}~T62-^7q zRZi+|)epYQP23r7xZ-tC+xA8&#umY&%b=4jQ4V^fdb^g zdui!EY0_%WYICe#J1u-s&-MMugBra6rN>zn;pHVeD4l&(;#!*R{<=q|?S4#7)B&V# z@8+5p`hkYu#OoHlN?CEPFH>jyeI`=nI6tXhz!`(FZXgH9A#?{BB&RM`3gZL{*SF#+ zEi^}MivFEc@Gy1!Ea($?c2uYR+XChD+_!ll_a^mD6W;65?wwh}XoMM(3ws`hdWgKJ7*rk})CylBFB*M5Mc2&x(!yWqG+5 zg=Mb8)bceB?DH

hKKD`{KS9%V=s5vC`tl7z1a>*j*A_LJ;0qNG8BOgbQ8tvyhH zh7|#;xhHtQP7X1aHyv&>K8tae?I~%Q-!`t-yZWM1uQ=nMgcw5nL@4 zh5&U1mo_C10@MBoQOj-XHrt|*=%K5KIzM=Y+526s#21j%*Xo4X*#0m-Kr4mlfPEw8JsnxGvYzuFy?)DIqzZy(^+#R(U%8Pnxc--f#8| ze>cnain6qBG^cepL7J_s^sVHN75Gt87#`i!W@R^>>1f=Ya=kaRJMMJ4FwRJntb|^5 z)5e(n(dj2WWswtk%1h(v<0Q+aWa&W@3A+_B`$=t6Fte^fJW*mU`MmBzE9J;x-Sp2A z0x#&l2<#3E7rtL|*q<8iK5NK?X5atA^{7)vuQYc@phmdipq>GS;A$C3)n>V#fI#kr zYS>-q`*aXq7cwt9s7GAZcW?XJc}+MiT7R5xB9;9_U+~Lq^N(T98~U+%f5Y%Xz2MI0 zNt}kSeVZ41=FR|4*}Q*2K_p3QDdDgJP88L|8{(jES`R+PQl_o7KF(LOs)h@Ym|j3R zcHVRYMj#b~&J}5DOpZvb6HZ6KQOg;VuO#?4-GtwXrt;OQa*VZ*1^b(p@~?pP#IoR7 zBRrzgO|u#NCc1DboLkN2iH=`=#6vwEkhJ+}7m2fNRPU6YJeIBaKh{yH(CtZg+57KE zE-+$lhuS~}F>UQ8K#oG}+p!J3hu->=<|vo!->mzqyJguIW=~Y7E`nd%3hfIHDO;_q56Fncwe z*OC2o4Y$*cjM-f-tpZ>!JLRoucqU>iM?x8iHrk+I;``TQ)lbFIw#m^_J$SQ!%Ys`_%Cm`MGbCUlbovi6OcE%mt73uXUe%XubzPI4D1*4yO{) z;aki@x**Iisf4^YasrQu=g!?VWRJkbR!)qPz++$$*+!x~vE$s>qx@11U&*~)uO@}nnphbFIv zcS#2)te{}ft#H$k+=zLU`)uL5!y|Vkzu)pilC{JA1@ctViX~)ZRp>%X4U{yXZo~QM zf_FRbQZIXb8d7u(z(PrHa{^tK54T)bFU~lxt?axYI$4Ar|0rM-&i+SjQ`ERglozc9 z8pJn0uR8G`r*bJ8=Fp0IL+n~$&r=lXrd(y3!*8@#ibf&hkZvOaV?{~x>O*{Dz33NL}CH&xCIH#j=jq=~oR9e7t@Dt4xCW`u&$Na)u9 z^0f`$C>*Tz>J7bdzB%X#HMmGh$X=zVfM$bd%blFY(zz-#5ovH+tS`?- zjy?kbuJqVX*a8+NtmHZ-wQ;a`mkvh3Jt^Em=iz=wa8&N;`J%}blpITA^78pXMJ!ke z9XKi?EyE;d^+nV}Aw>8xpz#LJ*%j@Mpk-dN;k)A`&;By^VEoV$^Mxy`@&Ls>U#uj{ zpoyjKF#yO3#w*AOB*zRuGy+rItO6m$TjQ#?QEMR>*d|xg0$Y8Xj~ux-UOw6O&Wn@r z2Y*!ht6jKKhh+FUz|^LG#e3iEQbW?I;8tGK{Xpyqz;DxDpRZ&?^L^uPdx+nZ;w4l1 zstjgXkErTMg8g)FbFW%l4z`Q;q&Ys{{LAalajNx5`C;o6B;4WMt*)mA8yB0h|FNTd zJantCvqgONb5*SY7L+x1r0XKNp6o_m-4I6u0N8W}7{nSANO$g8G1UnkM@_jMviC?^ zsFx-Ey@Bst-Py+}Cg)`rBOYA8_VjVhm*4PT1{L1b3adfd6h1BSN@P&q=9ICJbipYNnyE2KXd z%SjWcDOKHn*PscTvEZ}ROS|PYY2CsG5;b&X676*iGR_OG44)m?Tv74nXSv{GA~Z+~ zveT}3_3?}x(Sq$+r_H%!c%ef~KLoO7lVop&YokTK$<(b}kTr>Cm;=gNSS||X3k%FJ z<%??}R_dn&U2RjBL>pd;Gz|Rl?U~ZSfVfu${qKS%MD~f!p;hO&?9cYv8*OH~O!DS} za5mtTqSCfO5lL%k+rYRHdN*wzQIVlW6sXWSk)P-p$tQdE{{Rcq?j5n!b z#@-uu{uTy$`Mtr?jrX~AM^tutX|-NFRjC!{tjZ`3dXm_O=q!I5B1g5%(2!Zwlp5!k zBe~~j<-3$h^*bi3ji$8^kMTPufL{J;njdFE(a#2VARdV5s=lIUIca*OPhq^SSk>Q|oul)-TF2yyNEM z7LM%)9``w4Ql*VQ_WFlEO^SL%Zhc#lPSrAB3sUFORA$UBXDUCgS#}&3`iY=R z;4sKkjBBxq5RyvKktOEi_2w?^jOZDwfjCs)Uh2`n55C=cbH`GnjcPvoe03_^_naZS zf1_$MEMThRHkDet61a-0iN$`)!wVlW|GIGEOVzhha&YB#mA#=YuC=&Mpt>5Nu&EI* z3I4pWFQ$wkuio@|G6`hCuo)+5=f+~XK1_(So|>E|*Y#|wMqi@6%#wF!dvUMR=;QQ{ zuUYd2r5f*Et=$v7&R=U8JJp%JDy}^i{-dd9?1l|!eB=ERfhK7SCjUU>H#2UJO2+>^vZW?MQ_{**s=JD45)=c;X-##E`=4Xty;#!;Y zy&qEY)|-Ct_ybHad#H1l#L_-^ieoy`(qF;|GPn4 zv^gA|lF!eTF~kn7mt*cY0+XS{+~L&?muOH~;D0frCm}NnyE{M>7J$m6Q>B23@&Ch8 z{;zF+@x|RLJ-Ba%mm4f`k8ymyos1!xF_peh**6U1313Jg_6C)F;oS+VUL`1~S*wDJ z8V$afs!0|4AT7;Z{C9|Q*j{4x@Bv|{2uODyDHmpwY*RQ0lN^tf;;jC%o0ilRDN{KZ zi=74b<)`OU9oVTx_W#3izMp0epzzf6I)*q4t9 z1yg<=%FF1>4D926t(=xMFiUbCxvT4dpRip2Kj9~gVqeWFi=HNI*}tor_Vv*8q~{*YQGntsXefIN$rfOzMn+|plzon6~{Oh>;@AN@`wfV>Y z|HewZfBVk9Ue5b%YW$=O`A2zkn;%i?i}T;OWGvcMckVwh4iVn>`Si8+A&u0!Tv`V8!Hz%fW+}G8b^b$v#B83uw7Ksd4ZA?Zw)jeu> z_s-oxC%b23oIO%k1@Ez3nM&Rddbe=g^!6%2z98D%T<+Y1khUyI*R|o*`6oB0WhN%2#Qo;z{4hIPvc9)zJr4EY_oL(JD<`T4 z1Mg;3UkP)BjTlRh*nE{{8RV&%YlbR}IgT}O(ElTQVTV|dtB zgKBD_rO*4|anAG=H8m|Lk*_D-EO+1QL2tQF`wksnUyd5WI1p z8D-yl_MSOsX70W7Jd;1*dp;IfYyH})zxS=dl|clg^8;;U({;iwV~t7YDB^zYs`Y;C z817j^5v5{HAFAnhh}-MCt-E06LBC5WY)Q$#{J_7!ln9@bNmLcUFfDK<3R^b(-%|8SzBhuGddg~lRvCBKN zYX^7B>c}2xwOZpsnCRLk2vOS9mYt+7es9Y;AGxLTw`WA(43C2NU<2HR$8kV~j1MVc9(TuK5eUq-KOMpzgA((?bE@=lv< z!#Qe62f6Yu7AX6N{f1bYUK7tH;t5zw;By;0P)^sBwp6|0wdhMmTIpt>PUH$!5`_Z&xOLZaCs%? z9X<2M$G-raUwHAmO8--SoH;i_ByG^UyIi+-cK;W5;@0d^8Tnb#UP2r_S*9^ytDro9cG=F7{sZamCl?6TboI zS3Z5OmxSHIm0x~X{)Srg64WBt)wX=P&m#Z~&!Z!rD9f=iz8_q@Qh`jA%Z|zC4=P!u zk9^k>p;lZV{qlp6!}ObM3Q!~D&@hN~Hq-SQgCMzWT6Dp1XyWkm^9XdaL?lXxB{ z;L#}&D81ZXzLIFmJ!@cN8b>#6xSJ!4y#AwF z*`;y{yE5r>26>V2J>{G)-2yUcrFPA(U=6YQJo8*_4*fxw-;At%IfV`_K)c}q0wA?0 zf%#~3PgZ{Z4mmBKT4y*=9M*l1Z~fs}&+y0XL`y#N1_TA>ZABJNhvB)%d27}|GQMVr zcw6r~3cX~IVojx61YAtrl2z$AW1SsXgp|vH{OXa7!>HIbEPk1GT=m($$GRojI4(=h zCl^~IA+tKeX=PB=OS39{cIJK6y$a>D$qJI@06p|Lp3l5=F}OYGJ_s4jZB8c!4uyjd zQ<&Gi4;ziIc3aFm8TMTD78aQs)6z^I)>YHz>rfXsOae!gX6vAy&9s84;C33Bep-pT)$91>y75B;75p(%A4F##r{mX&D0}wBb9q;uI2nyYoh$ROn>Go1$}0 zSTUyj*?8SWsgd%~Oy9yIAReEQa~vZj53#};DrHq7JU^3ja%6Dua>ORB5I9nN6|4J- zs99)+MLjm%E%~ppFc=Y8FRfP66v(ReP%CjwOO~q z_Etxn>D-gcW{{P`wuEz}=2z-{wDOvQ-3#x;j(8UMh9YfQuL2$W%Ms&jU4FRyGR1MD zw?jevGYo7&f+gX$^ZW+e@W|*bffk5GT62H^F9%4wdj#@OBrgC%x<`6&aiqDyHPwFE<#hMo>TG)6h^OG?h~GGaW^fdX_;YT0@{Xe2RDcy;D}Mi>`oS3;0sSCE-4!`h>i; zW3#EYk^bnfw|2*|%^0241;|Xqz~h?tg16Tv9P|z^tC;XoguovrCHWMZFqrr8gD0FT zG54{(LK8QA)Y0h!0ahrT^4Yb7OwKKXQr|@JOnlvVLL}xI6h*TzzDsCuiPCv%<@!k`d?IF{f&CNPG? z@j`?0;nquLzLtCC^9g0|BPqOr6BAue%5&)-4ovi;MmhCzlCL2jK3!GHiE`tN*@mWn z@T}l`mkpb74}xAT0lN_TrDD_At6-)(+FuIDTVfJ>Y7CO4#i4ym7Lo^yOj2RzyS^mB z;}^R5>>BC716zr$GM72vAS=%|63d8z(wJr!#URz@4pB}Pd<$T-8XzqCDYQNjfNs}>Lq9$T!zfmXYRyHo9O?{QzmB92zg zg75YbX~T1qZvd0d;(5^Yyi%Q&ZmZ)6HGe+b+&I;$FP2Ov&?3fejErq2mD-0VQbQ`P zXUyO_Cqt9f@Y5jA^V!aA<^_56am5o|h9YA_v7gJVc2~Ea=;e0(q6;rh73=SM!st6s z2{cq~0rB^AW1fcRZ^_;W_OIDbWm)AenGm>hU3s{6q|V~LQ5?lc4RhYxurmtPcDJQ+wpbiS=`rIS zoz|y8FS!{U-iRhNbEP`E1@AQ0{KKQk-ai`V&2nN>x(p>|tFyTo9uEL@v5~iytI`BT z>nY-ofmuf?#H1QDD0U014TtCr9Qp^va9K9b)d(Sb-w58hg=dI%yLJ@*Bt8aVGb)Ex zMaH)c5zfah$~oVQGS8WhjxC^nrY^eT)a8BIe88T{c0}-t@GY8IJ|FgrEl`}JA0skV zkXvKOM1g`u*vPYYiXw~Gy4QPR0;uQX+?uF3qQ}KpQ|uEY7UKynSyas_K&fEyhcT@g z>h)FwHl;>52!0l<<8>p};Mu5{iTk6rc1ag1?FDCDZR(biDAnf}iYyzfF4seeFFii= zM%p=3ede$8Gw!i+m1v=^jH{AsZQ7vh)Q7z(%%q(+P;}?(l@$=@CFgyX-!|zeq{Eb= zo<_o8Ac%ikFoFFV1G$AjMj>dR|AP~wsMl~zb~d&03buznN)bFF%Gs;EE)l@XNzMx5 zz<;*Da%=;6KUyNe`K_)G9jj<>JyNBQvm7w*x@q={9wT1}Ou=m2b+)v3)ltO3!S#|s z6gt0NZ;|G;?6pscYa4Atd%qY|@o7KR)tf9gz>W=aNWV(4&_QSOEd<&rV{#6ENfibW z<2p&CMjf;FZS$Esoh2cS;j5z#mN=LmVJ#@dsoz%TL0<2tCJgRzWRj17@HM_Mxu!iM z>(|2UL{uvGEEf9Gaat#bbEfvDs8NT*KaK~r5Qh{3=K>^2wg)s_1e};3N$SA0~guU6z@!B^T-#CxpG$%^Hq|6 zgVQz)i(aKDOD@3n@j14+__AulbDT1uT)Kji(&(IKj{%!>bo;{U?LAl)wq6gdSSfN* zY`gqPgpmP;6Zw`5+4oZa#lVRW_#ZK_r{@a^M`DOGdh3=BNKmrJe3Deyi>P_lfB#5bAEa9OXK;^;i9K;g3_VOIav zBhi&TCDU(!j>HG!zydT8%T7_&Sk7!y`?r_nAk$P0IL2|x#eW-mDapxNP+e_jLP$@f zCch?ExD=ePB}G40Jv9=d#kH+^Yk?r|e`K%f4Q_APqrua{qSvi)P67%Dtl9{akh<(D z&HW_Cu#R_NTx&pRmIisqPSs+rz`TTDBHNCH%-5{#D^pf?(?cRbalaUr3Sj20z}mzZ z?|fJVxyU`L#8kpcbo!LE*Z9~t=)P=VgP&wWcNNUPr>rpi93281B+J)`Lkg_O!>l4! z(V)>tKHAmkW=WC_uU>#b6`l2yX2sU@Taf(^DIeEzlj>^I-3sMtitknux2kM503;<6 z-F$Rz+6dYFh$iy zc2HhfTs+(dC50FhMMhZ3trj=hV<#u|8}q4!=!)%9EQcFrnw(cFO8SLwLH86prIyNY z{Ce;WbM5|Ln>FNOQNA%qR&)0P(f4!_s01gOo_B@BEy9)h-S}+vh6HqfxZx7Ona79)|b}jlml=ZT$gH^a>Nt z9=>$e7g$DudApe{E`|FL=$chI7y?xDBNF-|4YFZSngHg2*pz1tl1pWHCLe!^(Q`b8 zI{teC#p7;_QX%x%%LL96)k2=k+F{m@VwvEb33e?6)VdXe-X75wA1dz{^Rb_nQWWPm zjw|K|-=RfZu@B+xrw_(qp?5v*x8r%;134;{hf+_viDWo!eFGy$x3wHN48f{RZW|F- za&5#p{X3VHw4_>Qnhb+%th!0S+*GRE(jbwPDjM~V?vU9;lD6iU7WLvaalDy{XXt-@gnZ%`WZdhY$$e&OJh zdP2**Hcg9(Bg!CcRQxi!D>me#eR+1|)!&#d2;UyMl|8@--BYKZ&*|aqK``-V>BL6~ ze6b+Olw*(!F>Ejg32Kb^J51ah;f6D|T6Y0G6Ox1+dJsq@H;fg855iCY$*7#-{7}I; z_>3~VMHe?}C=eNPg(EjcYV6aaHnYdnklFh9FeXFVpf?=CBv~ITn8nv)lOb_~OqZP0 z>6=OL*$Oblua1nEs14+!wmfYkzhcpv5MYV!C5TZARbt>(wf9bCT@@4wY8H;>kx+_h zNN>|D_f=gwjP|v4-qkoR5=7;%JIN7mcKm}l;{&k$#Y~ED$t@1i_fa=y#E$qx#JrU+ zEycR=x@Qqn`8Fs^_X{hE(eiE4Qo7mqtAw)KuT11{CGtA)iGs-50|yoB?XJr8jRrSq zrA6mM@C*4W70!rQ{+4nVD-XY@l{V3fydwrOH@NR!sEp=ux*`g`b>~WOrKpNc1CvIe zFy}f&kox%5;;ql5hIq1vLnj+0OE(Al(G!HjIVlUV@iZ{}a*!qrOFmo1rJXB(TSZ=7 zDm$OSU*<9|D7EsMZ&gC6&qo*MTX&jO-VB%8Q+KRZ+~*Sp_Ok_V$A7??@p(47Fd5YP zCE9gd<^_r%BQ?w7YfB@r(BXN4nPPy3ZcG?IoeN<{%aBZxPjac?b>O?G069hoy;DC$ zPyQ@9IlX4I4FhD_prYiwu`?632eoGtHs?MIXCzg-9$(K_MT*V{mRg(3q8=bfEvxYR z(YurSF!t6jTdTY@J3Gk_iM-CCAAGZMaex^<(wmER+1N4(E|oC>>QTq0Z5fqUAnsj1 zqn%b`_0Z1U z28MTi?s95Z>zLSf0`v-tI9h0m_au}dtypn-u@GLaNRa-eX|*wqi0}cS)$53=t2BoD z55?N_)RrU1gXmv>T$d`k+(Oy?VBCh1DL6`PE^h>6E2aV>ZuCtL-c#o{b}uXKzb(M! zJ4j_Z?U(PzckNEgjf8-Hs3$*X$d+T!D^N~MrImol#9%l{3|@)v?KU)7UfjmpICmTm z&wroUy`cVEjtaUvRm~}X##BURhdxEy+MssRSsJTx;LHJ>ELp95H^rB?Rd83fC)HV( z^FbRZHzS)X6`2sOWm^4(@o4}Y=(sNojee!Ysf;CV3*cr)2AM4w0xho6e{7|>W4a#`5lk#p?EygmY6xd4*~; zgk5XYkjNsNhy_DtTE-^qtOWX*{l5O@#_7Sk9>%>JHZ6W*oFwknl%|L4UAz~0x$lA) z(4>T=lLo1^#{oF=_|o@L;?zedPO(*?L?EGN%D#O;O7RM5V2qqHNtwjWwo(k!!|6+X z`8Q&i1K5Ro<=BOyCUZmQwomnsbZ;p0 z9r9HiR69zLwyvVdzkV$=wrViiJoOi;DyZzGITJRCim~ zVe8q#)2qMN6$cfHyW`}H_aJYn4J>$WOZKyEGwE+{m&Pk^efhuJoak4rH3DB>+7cW_ zueUm%E9O-A22g#;M!8x?FCQ@pZ{T1h!{hI?`itbBH-`%)QC3r~E7i@N-QP(;^LT?h zu()`fhfZz?J^B~Gncnd4nj--&R9MAYl;~Fr8q{1=Nxv5X$x-<<@cA5FY~M`o*gg7~ z3<@GJ9n8&20Vc9`N&xMIi|;T3UZ#U}*Pl7Qz(&teKWJ0DB}OAMqR>_`b4r@01js~o zmayA<>EyZg|6Q^DztUnux5Y&4$!}#N6{~)8RcD4NGQ6_iMqJ*V3}J0r$InpQ@Pk5` zEk4ley#k8F7luB2C%bJ;JTeb6STb$UjMlZ$3sJGjgO_0L@Vo#B-Xd!2)E>0S`Rj<( z;|JdW#GB4zS5X(g272Aa>bKOkeFNMvehxS{JNesp{SrTaL_R%z6^IB;clvW#k6|q+ zxjIdLNADH!KBMnZ#%Eie8^^k7FK{6&++O$*mb-|d?PrdqJlslszUmys09nq($4 zqPa@N`8F8f6Fz}D%`jDr(!y1E$JtxgvP%;ujX72ImTpA$Dxk5B2n=>g5z6^k)p*o5wA2G&tG>H$_K z_mZz)lIyQC3UUzth#@$S>aW9WY_{4!Efgv~mAiSlq!65?H_$hzVP*3IaQkmaoxdjX zyYM9+?|8U%Yg12k=p-pSkMg8m=@?;Q?c#Mn^z*%k7d;+b7iL=QWsW^GGzv5;NRWV& z(NTyFwyP4ZuYfU&v za&;iMyOx-|{Y@e$#sl%iHevO-B^leC34L|qe3c36{b$6(CxaU|PQ9u>^+gpUafzaT z`rU%g+4JpQX8@x?XGpfW08`HPsVj~2kIi$tQ!bM znBGNXbH1pC^~<1Z%^JW_E8=;e%-H9)c7<$atbj_;y^0WqD_k8NyjYEaoPr9X0)9W9 zH_P3lzA}(q1ue%FVY>`^JwFeCG%zDc~m!1=%97N?X+SsuIRcCLF=XiV#`%9ZxgYgNm`^gllD?xH> z?02Wd^KO+K=O?Et=+@ckz=e56u@%rL7GEMj0{^HzTaN-XC#urQgqL_lEhk7m-!yK+ zM-1yVB34V6!@t^!ug@50>yHQ6-RXZM|Ivb;QQzUhR-kzK6&Q zgKgZ;wdI<{u*-d#&uft*X4fgM>9TwSc*J}Iu=WnkSlw}m)N8xq3DP+%srJ!KQWy== z->`W&*BKTb<}NqKM!|^7VnZEYrO*^Y>0tbop9I0y+e8uS(_7 zt`Oe7;Dg<%9O$-mf2FU8eQwX8S(_yJVMJD{)BK*=xJa6RxKgn&#}^b$LP||`S}8z;<~p6^yFjV zbLI_nKx`iiNfO~W=;2qHsdjhZQgV*t3jit28zRSqVU7JmiH7cYRD}J7GHd1NOnI^a zT;4r-oZv90CND?koM6}RMYe~2`YlH$#$vT1jr&u%e$1mHLj-s9u8$xt6J}R_B$R&7 zWWuitZ7GBBf+^IJ>KMej>fH+VX(b+9Oqlfs{2uAh|g8DrmEKdni2Eb9#_?f zb<>^uzSwSE*ilf9CB{JuN^P#}J z#W$mJCiZ5LH&;gOII9WeJ;_A)H_Jyx5w-byZomNb{vw#ido)6j~c@_vE8woK0OLgnpup>WJrUhYuj3$HL@o~ z&$NYDk6*m55x^&4dS~LIc30Uxb*@X?en^_VsP}_7n9VJ7s6nS?71wBO0iQaQGv8%s z=K_v&Hxuud{qea}cyE5hFX8x&tZ9&uKTt-^sp@Lm#B2E>}e(S?@Wp zuax*Kw)oRP>5)h_ls>8j8pku+{*T#Yrs><#)u;!foe0UDK#$9WCS_~!YrsU+AlNN@ z*3*XJTXZd2}GJ+gcbR{@KXl)ajsC3QSt)>$1woR3liQiqcPHQwk zY2KW5h)i;g$ahbhTI}Xj%u`N$yW~dm4RDr#Bwq9*e@`PQm|?+ah_aJmJ4%D%5wI>` zXbUVaS`ri|%P+YxeLJBnJ|$d7rVjPnPa)B26LZ~Ncn+DaAK#&ZuPZUuaOkj`?S$>19R$Wq2^C_j-*IlnMz9udbPV7bGH!sn-)wTbwd9h-Ks zUN8Gow^)y!&hQfx>2vSS%kR#NoGVv<>USV+PWKQLdYfJmHHDtif4XKh7^SelyG?p=7@|xC$)j_q9 zMd~Lf%dbt($0SC3Z3okA};p-Qs5!} z)---KQrzv`wax6@O9r|Hnz~fO9ZwDpU3zX zVRe*bXU*O!uxhiIQCafg^F}bU8f??@HvqEcyYB)XvsAb82}$N&~eQ4 z7?n?;Il>CB=%xC5-T5I-4|wXa-P5nrlVdZ%@ds6Ebv!}f48E0s__eUh@$Y7po{z2n zi5*J(O-O!~{~uWDf7n;Hde_)d5Jf>n3J+SN$IaK0yv=*E(pA9z<>FN4ZsjS%v<<@N z9^QUVwVufBnWyz8Bo#9{Xlo$~)o|_H_;TkF;Ed}6aRBuei^jE8J9vVV)gCW~%4<`_ z2N!!A20>NI!lPr+mhbb;6^x4}%z{eeKT=nbY`AcZMw+9O_2TX0SYr~5y+skCEapa8 zU_Bf-2=9j3r#r@M3lKu$`Vc{&c4M73?bIsSXZxm}Zh_0qnM%@Z4xO2vZ#J2={7MWU zJRfp_75A&OWIb^8tr*^%V7w81a9deZK;JJ-_(5Y$&0r&TwQBt*qXcPKgoQcuYvX9f zcuc&;8{ZB(>sqA@8QpYVDNHm$7-*|#Wn}0N90x`^jo#u#+d#XkBcpZ3I1?jHn?Orr z#Jopgh^9PnPPm)9wW`vS)39hgmu!e&S9-phM|FB^r*GM|P6sIx0CCyt>RqKEa3E8O zIm!?Y+K;W4%Lf+-DZ03pB{Uaqz7htH&wwX*wSvoYRlmY`(Y&23J1LVFdy9jl^6)FB z4vIpQCtAL!%OB(xe6VEATB$HA;qi$%VNn(J5%t1A$se+9XdDdFE^xc4;lRV`dmVJwwBg zmXEZR(U4p0_;QqTSxM#gmwlo8^jtIzRv*Rq-Qo6?TxJD0^UsuPWO6-`TI?qQ6cmvs zQr67{-tH_(p3t6#LBUdd^Jf`Koi;Z$BBG~k5C^)Fp9kGr%O?B+AUtpeIO6@OH@2?G zdeX0&H^UWgA&4;=-s0}I!Y}Gx@sVrKR}7%I#pkqmUGQqNyiGdC_XG+yjtDvwik!@P zI8J9(KK+G?+v29a+ZWr6s?5atf>dgkLaxJGm6Fwu7j(U~!9x0fT0n&FZBMZ7)UQp_ zB9HmAoyt5Omg+0&Yu&)EsRu#gF4&t!prR4KfvNktcB zaX({OzV(Od+xm_U&#ZJ`X5YjZf}VuHJL~BTZn85J(eKx+?($xsxOPvAO;z>QqXdw$ zUEc>d$mVs#Z705NDO-CK^On_lb3j>OsxohE#B zM0A!!K{1kQZ!-JU@hgqS+Mv6^;pWPlfzQHs{z81YZ#iWCq8`4D%hyjy{FEYmeIC$j z1?ipIO!nuNxHK#CSTm1w-iUeQK5{F1X^oIc9i0ovf_3H9s$axM(v# zDDc!HT8leURbWt?&YZt(EE2boPp^o3&ZRg9XZI~95`kU?Xp;HT@8=j0a*Eg#Ei`Ia z=ia&U*bu4pRH4!43Xwz(5H?o)srCi0l%f(cMFeL{$SIQ_o?%_ad^BEet_@rZwI!?c z(4DAGTqszW-&B>$qjAines-Gi#oQ&eFn<(`{9rx3 zS2mPIr6PJkAwYmX{}r5*sZpCeo3oTcp{6faVX>yg-0|Vh_H3ML=CvztL^5eZyM|fL zb>)g&oocgXqP@o8ok&Nf8Mz`MOe9a(y3x1z5#GM4=~>#eGsiv{t!d@X)mtc&A)LG( zx1vw0Sb?CTT}5O;-%o?>O5y&d2CPH2Z1OzfH(q0}ibU>FxbgT0W5@qO|NPjMDevpl zgx7}&TKm!O0|u~$f#dQYN=!&Ktlv|PHXx>4%86{!#L27OK-<11^_Zp0a4BN6U>lg8 zfziULVOuPGm~l23nAB1N5mC^a?QZb1)6zX9^uGm$PZZ35A&q}aPY)S4UwJMJ65%eH zOZpmmPx-5;+If5CH<&&Xw9euJ?jfxWL|By1eZxG-K>xzM{WSr|}VM+zqMUTCRh7%KGkNsVKovxpgGGI z-mfRwCVWT2UpJlh0oS8Ef1i{QfBf+zEL2&*wxWECL39bZH^(9{pR{AI_IVX-K^T@&bwB4uD!Q{QZ7(LlLuY=J1!?bUJ=At6~raB`0;?g8>b0c1v*%u*{!ek6QhpPMX?vDo=H|~gD5xlYWN6gTo<<51ps8+E;B2&mT{)zX;?l|4e1QR>CVtz!P(08 z21Yi&_4Px2(us-G#zmXw&*nmU)qCft-!)h14zEmBX|K#7n#yl&E6o$0(RHNjB)yc( zZz8?DXInPRZam%(9|aMq#-uaYED8POlrptM>a+^`+4mBGy2l7PbZd^VHu}A)F3Ff( zltO@daS71IN-rx06=#92L@1B2NEMhBYKvuIru5Bv49hqk6bqs#ndDkj?HS_L7T?=O znGejP&8-F>8m~%Tl(vZ~>6d1iqFwVpu)Q^?PhX*QPp)tRh`W(AnbhKBD9F4NOy~`#BglM1`}ynGmV*Ve#uclTP9*gj$cKCL5H9Nh@LPF1XSsX1vuzCkp1%)kRSOA6UoxCW+qlfKqn zc9w5frW`t~EyO2G%Xx*)PNSfgmq82|T(dNtev{~@hI46i*7=)2uWey%cNNb-=U?3DF z0b`P5c%FG#1?i3ZajnaLHLMW8Oz4U|@zt)KRfc-)1xr`7BXHCWg*8Iu>O&i+E)>m+P3wF6u0WBDW7MVC-k=;0qpp z0brB_04@fOd;{2)%Q`3%mrV=rey+-rS=6`J_xnz~>_Y(n+J`Ry0MVcy8wn*Lrd@j& zg@a1*VUEY1W@`qAxwy=Q_G4`#^4@tPK5X@o2A?YT zS$by6mk>RW22LxoEHYnJU0gHTs|qsL8g*%!SAPe4B~dV~T_f?bfDVIQgE|GLSW8{Y9o=UKE`1nD3@(^X!9Cgfvb%;j43D8mF)M5eK;3x1Y` z^dB=ko(9hkeQm>~{R}Mn1@z;55qGJ8cIqP==}sh;tw)u80WttbN(PJw>`1 zE#lDVzA%q$8-^ZKtjZ*<2B}jS3Z_-cKo={=!^=DS?kmo;8-Vx+Jjy-Hf19%o}> zL}fQ>x_ORw@X@F^sbOWdadw5Sr%*y(FPDUdFsmOUP&IH51W{@0fp;A#V+MaUE}eJl z9P(x|t^6Pz(8@M@t6>h?%cSOy&uJ6M!ncO8>rjRX7-kLAx{vN0CP%r(14{ImSR-6QYMBAy)*72CuREkaBeo-QPD1YGPTS}Oo> z&7PRSHAs8CL*TeWqB+=t2%B3V$dY78+;WIwVX@w`{stH+Bp$}QYd!^>^ZPAg)g%t7 zM*R}0mTmPO-HPiTJi{*xfj6?}q$bo~olv!w3saEF6reK7f)cmbj2H)xl!DyNa#y8L zh*d@+O#L^&;lM9T{+&AVICdE=iiN-rjqvS&-eIuLtW`(#Dus~mV(FYl_SOC!CloX<#e4a z8n=!D1})}?GpH-}k@+_)(<4%Oy(k#lxy~mPyk(GpO!aeUbEwrV6pbV@Mj{)a^KD>b zrA>O5NElc^r4=He?AS}OmHvZ*(?l;ew@o7eJ$EHb@AAI_<3Pi=iG{F0^fzeqM zNm{%e?*uZ7h4)*d+$?RB93Cf*W>KShmL%-IVkz6lC zFE?Q$9=Ys}?=jsXu}O1fXKg_y!e*_NWUeqhzqj_mM?3f+FZ0ggt^j94rz;=vg!dcZ zZ5t}{ZFlf-aPmz2Zp_>0-NL+a#E=VhaO4a;PH*>%?gN)#SOM$J4+3B{3GPU3;<#gw^TK2Z&Wpx9w*v#fhw^$wu*MGqT@C z@<*#z8ofqU@jSG_8#52>S9JI{}Ta`+1uU;%Xe!Dwq@^(t;Nm2sZG;m8h zRrbm7@zlX4^GZRLWiAuG7s9*VPp%DTB-?6=dY5SVF1J)6D?z1I!}tRcT;O5NaP9?e zmY!ah@f)qdP9)y|&b=p+lk~A`)Eu z{def<3{FZ?vF2M*4-7uWh-$c%Xt?G^KewAn=}_EOC1gRNy4ga*OsEovl#ov(Zw3^~ znDV3*xf!_=4D;y;_ae%B)KxVWc?O#lJkZI#Kp?x&dg+?K{vD@gZ+dx~tjH{E2Whi; z#%xztHaQD7pQa*hgMzHwEwrVGswq5vHTDtUqQ-B;DEQBQ{!0%+h}-F5QFS9W*Hh#6 z9lNWq{k$QS(E-}a+&YY=(;Q80T%84+Dd_mU2k@JfK4%9ToI^^$;NB|{axY1QhzWmO z38All2|f`X2ib=tS@&^U_f;}Uq1ZZcyh9Omk70fp^y;8{9BqVWVd^UGm6i`S#0pOw z?;>S^LDDuidoqY86^H#Bik?KuQ&LN{WoD9oVhRKC6a(!Pb`J3k@R@iJQe#{&XR@qc zKh9&`Am&tJ)3>bC8kudYZPCB*`aqkT2(HKRjyqDt{iTxv!rGGqq_-HmNlv~b|8+6h z?CnV1LDzA)V@J^^8?Dhtt z=?YmqT~qW z6if))`q|{EUa7{H1h1|1qo@5NYq}={5BjGE1FKgx=22=nM$R>CxqLzF8HADul{3K=iAbav{1s@Px`TKtu`c{B^;G8X`Hd=OI@W}JsR)XwjMoT4%VN*9+ zy*R0)ZFZDI3LK+15?8J@tb`$gi5f>zZw3z?YGQgtX170T5u@a>MH>JhC_^;1Qzto= z^iG#Yb_D%2&?+TMfzBLC4~T>~zEP0-AYp-r?7fhisrng}xg`m3)Jkq8m&;SvRFTpf zH#<-Rcdq4r10-J`_4{d64SEHG_JupSQv=b!M?%Rz`%m_>xbNSJeCqK#k{T{ua>7|O zL&~tlOWSfD*r-YW(a1+p%^WgsD>FnzYa=;w9b2CfBh4@6Hi^hJG+&U@==Mmz3{q?< zLx)YAGYU!tmJ1;x$LQn)D7Yb%6wL4b{f5=+TUY24{QzEnY+PbfVJGfFQf38p5oxiIn;|Sx8|WA?(W*apW?*sS-%f61m0YdRaxOlh%67xleQti^(?pF z!`UG3sl%#zVyFppGJS(Lj6w?op)BwLrcjVbwDWz7jB&94UU;Fur`r#2lLZg73>up3 z>k{;jE$P0vs%zc&lRilNi~CFU_^O#9rD3DhR_mi>PBrQUkZk&E`=UG{&7q6w=6GAN zDSF*Xe6}Lio?fLj+V9{;FQ5MOHN>h5-!}lX_L_vbF&r~ucv~!v!Hz2?T@j}fH46mF z4qx+>m|AF-BCh#)qVe8FP#dg4G0f$c>iM6pWE+25C( zTBQGjjTR_6g2Iq1xJ>+#zkd|kC;j5TQB@#*6DIT~s|wiANK;dJv^efGqRxCrwA{Fc z%7&H8f_Ps6|0jb3(U4OQe-E&B{QvRLbvNVZG;v^Y5!Y71uEWul{m%3b9e={C#JhEY z#+xD$lt`W<_1rSd2eErK+m#4(gJXtSS%dI1B|)=N7!lM$^EV==6H^%IQ*dI0047Sy zKBw=s0LQXMdA=S*5Ywb?|t6I`mlm@G;5X2 zn<>7Q>AQFVmEHS{R@&bAQbh>087WG;E0kM&z(ge5W1;#x$q()c-0CNZj)Mg@7aDGsV5(KeYYXnJNh8ltKnibt2;b^~+Iiat1^mIKfk#n%T8FgMCWyr)m z+}*SJh8#5W>s&tAoZwCkZH8Y+3X-l#6+LDhiFni&!6&Mb;X7Ntyc`@CAn>$+CdeT? z0Gev6?#MpJ#?G>OrQiEr9G5#Jt0{{2l1(G?r>1ruVc@JBM9DK^S4k=Jgk9v^_$6u>u}HsycsRv_ z5kzj_7*s0{P0XHcWvF(!+m8!-Lk4VEXX@NRWUgGWx-j|`xwVk%n7n2TdHCb~#hBK$ zSM7Q-mlyi`bkNFfM$Zswn%onQW5nW&lg%fetZNaFTR-9_XgYlV)lYAY|7e9;M z`i*vf{Bu{O#-1SdHl3ME=;uR%*6x{~Me_kBGcKuei_&A2S=B$j_u`MDXYpr#G$C^H zzkd^Q#x;)8y3N=uV4}|FT{wiu$(qwN;a5EO?zak1L^qS?HyYXCxl6P2c?Hy({Bnbf zsu~@Rp*crqc18><`q^791T5?F>&krtFjODXf+ptr8SIbJN_4~I+(ezIc_Sv`nCWvU zoA=@^)UyZgPoAsJV%02;;R#Ym%#T>Ch8~vxFq=PO(Eg8>SRVb8wt8LQAJC{z_-SCI ztKYrVnA<&0zxAi0=dhEG^!(;(c?|_`LuY;2ScgR}uYRW?rnAU^puW=(hm^kF?IUCp zasF=KfK%eR^CfoIHMd-|3)z_NOy*&hR5Q$B0;4IN7%n zKU|eM?T4Z0#QiW7z^Sr&FK%yEb*BGsasS+pvS*-xAFcS4q2p957;;S<=`ZlvI9JhE zX`p*ED+JA(9WSpG(fRfu>;@UNIigv4*|l=o{QZO5+Jvh0ISQ&`$kN z#M%9?nxUL)RU<~6oK}=l$akFi>mLk@awxy!%=Z^>DPvS&3@- zspeC@{N+@X__rd;8Me;8#K7If>oojFDYo|<^{*3;$6?|Xb?=Zzr9s9nx+5p&^53eO z?SP63ZlP0#wic#)S%bhZRBkx&urTdkX1=FKBvT@z^H0e2|1Hn*U&;gioNw?fF_GyP zJwpByl~P^`4#Vg&W9Rcu`PRN8(b)`eVPPU%R@;eC)U*%gQw{rq`t$pEnZ<5dWXLXv z&MwHpLSeY~L};9u6amL5N#6|p!Ik`zHJR=y(^B4^nRh<@Q`4U;&Oe_l8VHWF?!eXp zL+9~!g9WLOaZ$%ZVHvXJWNKEPNr-rO6j%gAw^Rc1(pp1@41Kav*q0f*kN--W<$1ZV zh4IJJoeH8x@I_9kkUHDSGF2WC-=39Wk8_1jL6DwTG7XX0xy4Y`2<}m4*9xgd%!J0HsvFevHj@mXPTRrW5=--MU#h04WkU znhpv<=uRGUm($xT(3?Li1U(y8u5|6Ve1(gieRhgTKo&X2tW^m~NBturw>yXr)hDy^(=xxxQwP=m@rRc*(*Y#6gyNCKh9M%awxKMQ z2qa|OX^4D)P{4qIMB8MwwY3q1g}*aB+x+Y05Jl1gVm=lIEWE2(9IH<*s-4<7^wVH} zJZZ(9G10ZGVx%S#N&EkJd;iCiRve(;V6E|-QqtcwWM&2o`*5tW{b=bLqVrxKaxoXv zjaqOqTEJdbI*h1(ouHMW(y5!EdKiZOt!|ay*s`y1bS=N9SJJbsJ;HpZc|JqsCC{#}(5@}Sv~?7#uJfT^CPnz@Pkd5T&%Nl~{Ga*c z%mt5$-CA)x&ZT`(d(nw3xKTkp=56`j`H3v(nplj05!xsK6u+lU6zR1~?{@#5VXb3y z*mSX*2>f38gAzIgU&itO#oc>{HL-Sk!|1jvC1(_}Ex)CfdYYn--h=Ovc_y>#Y>A^X7_QZ_IyBhU|Xws{BXO>ugzymRrK7X{-uUI3W{Cx1vzeIe=i|w<>8d|nT=1)b-m?iL{{_* z;}fJ;Llye2BnT-{YIKM(FKUe4yfUka2ikmd*2~kW%&8JgWxz2gK{q8QH@>7g+#=lQ zuS?gojuY`Q4ayeg&sec~!RRdCGk$=A9o*FTdetKSry#?m4`HU)9(;P7sDKGX#}oe0_~ZKwZCM_v~H&osvM#xV8VzOqZmguYqj z8N2>U-JnMzn@dMJ5)+)mI(pyx86Jrf9|J)Ou7WlDt`$zst z9{;gM{L9Hp24ml68!Q^Z9Y5WgR(CNBlG1@jaPXpqRAQ2sR34*fUpJc=qX{u0K(o!q zhNMR-h0WUeqLtT}XCMlFL{^|t#xP7pO@5&FPifxySJGVM?$+qXQHw;1Q>cXFo}$_p z6TZKQ`|f{uj)xR6dD&}XAV12emBQj?3MW|bjNmUlEzC2m;<`%!=^^vo1x{gld%C#e z1CYs%E7=AbG1WD;*_w=bS#Z0F$jk1|FQ*zW>b>_??@2jA{bMgjr^X|+!Q*s}qs6s= zdWBR?Qo1sI03hJLX$U~%3_cGP{l4y{cy33CogO8j?02#j{99%TG*ASs)`GZglIBy& z0mg1qQq7I+V*70H!{>4iLS&Pp-06|q5>I8~b^nE}!Q{mHrwP^hD--J753TX8H}4GB zKeUhfWXL^K-zL;jRW#xZ7cS!~wS&4cTzrQsss=6EPVcXR?^Bv{*=BqMKjXhPRRq2Vruwxg z{%Cl1%7u6jHFy)6p{g5k28zx2aNe<5Ih1>c-l`u=n~{>d{Y?MUKL%7kA$7BE`AcJF zW#jdpmcd>z!wz-d?+bC@+Y)+@!@rM2{>zU-4bE6|^2*V@q)soQczXG|lKpnjKlZ{a z-&;u4mPD)zrhoP}{A#A0bG!GdjT1ory%d(QIs&a_ERe)$U@Jc`N?GxhZuwRI3pL`i zJa{ev?h7fB-(VDO(&wPauiSbOcWy?YiVc>@g$d5rh(|Kru?_!V9Q8d|MiCv7zeBPR z&In#YbY1kHc{a?rQJv!Jz1+r}!uP%O{hI#O3F`KLjF9p__S6(P=BuyMlaK7)7OeLz zuS5^1CA^pZT+-qUEyA~p@aW3UM&V!a0DHoBdcvEkVkN^Gq+=0u$4{h)>RlYKD)rYJT`>* zgNotn=#6nf(~&-b&y4;eIvSQBL}eK#7lkjdq?B^iO2xCE^?Wv0-M=)Cn4H+*KkWSf zR7lTmS@tfA7rtacyH!RXP^xcVMU80rd{w-v+nqa*A z19APqXv$a2WU91*NSn|WS3D_4iY%5yX~P`H#e0?E%p#6EPe~tE{Fd=0AD#_)n6=L5 zZ)+u|QxLl7(qE#UryRp0tgaRIDE?D-VFw;oCI$(v=ArWhwOL!`xPIqd`53gJt>6DW z);S={ed-bav2evi-XOo>{J6Pli040cNixQYIDy0*8L zevI~mQw@icPj$*NsmOgjdP6XL+^o@=w(-tLevx<4$=Rya%X)E1lNrt~%&X^t@Y|1X zbbH9&H;t$VYVDtjeAfdx%+|klSnl{i8Rodbd~;^byVgk-Hwu55>Tt0<)L4Z+o_Q?v zvZ`81zaj-*)l!-BOVJJ{KM5a*kOOjR^68s*$@ssBckBoYmwb*PhfkGT=_KnV*<@!s zU<1$R#+n)8?OW8IPFFwc@yg&+boA-ik~#DlNy|KW8ck%pRx~CoMOWb64`4=gyaWq=8KA4ij@PF$eWmXHATT>Mc5f>3D-)zE25LCZ;#{C!`zjJ;s>R={P zt=JWqLQkJZdka*3NYb_rOT={1hq;6vX)FNpJNG&knlypdEv>H_?^WSK1bX5j_9cy3 z%f2~*U(pu7Pkx@eu0_f;GAr1Ty7y^%Ust{7u%wZh8 z+aFZ9X^$RZycJR2e9yL`C4DFejCGSsMQKWKN2q5FZiZ(7~X z=dkGE2SAX1n9LcU(mwxrB(s4tv|v^%(L@T}y>wa4_u!2$ z+KMeEEVDd_r97vFP?iQ^By zxc1jj)A2+Wama=Ia^ey>Y&tpasg3d24b!HGH0LNI2OU?-jr5(&aHq)Y z>TXghPHeH@+V`6|KI41DFd~@^B!l5u7fZ1;fO7F8AdN|PZE4yJHE*Y)8?|m0aJ?BN zQ9e{ciua{`cah6FdOB{bEe6&ShqpL7Jm@-i%wMhK;IknSHctGV=M|L1hNGk6^uNj@ z`@HWt_Vb(3*6&oX+?F=$oV@z-9-ogVsNYPYAH@1KJyLuW6_HNLd4?@caLi-lU#IX= zt|q^CVpZH7pPW%#hJ9b!NXeGY#Z*S#8Z4DC$njoRl=+sIPt#K}a=-1V95GiRXgNk8 z?4DCsqcbneJ1c<}y>Q(+RujoM?#c5OG00>V-S@N%RUw054N^g3q( zdsdZzlBwHk!`!$#!3xb@zPNeO9`WqQaxFMMLdo}@6n9a^?y_waBXX3dNdm-7Z6seZ zb0?8EZAiazG|Ac;FQ8%vQ0@tDFBOq}lptLZou^+}uE`u%Okz*Vy|LwCN7^74hq!(` z>b-eGy~36gt%+*PF4sU4CD><`J5WfG-JFW_-6i!B-au>5(vA*WidJm0drozhhC;$v zI_4c6_mm#9|inooS$z4C{kZ5xR_1d;J z>Q_AegQ^U!co0b<<)_(J2r*8nx=HCg)OcBsbo}Hi1XtVJK-XgSNN~t9nWK0{8!_NR z5B4jh52^;wtF*e0^>5QJItP?WRvV1wIB4$6 zpT~(nR68wP5Hi>R5n6L;hfPU6Is;wW>%3dsgzu+BA2bm_wT?&XYGW>=agHW9BCN{_ zLh+*4CDCA;_}qzn0bZM4TYs&XQDe*a5?x~nu3Gn^Kd6>??aQ9*P6U?d<97746u*{a zrzrcff4q43ZKBhY;ArO76FB8EZLfCl!O1Co37)U?LM!AncAe{mUkL3OkU8=FN)BuQ zE7#cO4Ts6kig9h&zlt*xs2G0w2bHAPcIL%3c9y{5Jv7?gVIum%dKjBdpWRjX8|z9H zU6qu2LFXWdF+zxsE_Pm2j7xkkyv2UFt}|KkOX*f4XI^>^ZoOHeVpZ5Qa%_H6T?8{k zHWjyuSACn^GEn=C-uf{K617Mu2dK9$ut#J{`X_im5DMeTjFhG?wju0Um;s;?k93lm zdlP|E?4AI(t%_e67i&ZFV@KE5ufr#5+icGk4HuhoL~RJ6kbqJ{!m=x0QMU(iJajA( z{p>dhrsSlp+qIWx5>3L}o#*g@DKPF=9Ot>w0-Qa;!Byvi>o~qe@>S?}J3OblbK>a3 z{*=A%?EbjT<)vr0YWK&cq-*z?l-sK3h7_Il((`wt#&DF%DftNw{o^Yo3nveEarqQY zS+3`bdmDdoa|oA(dw$+%Nbz)d>-_hbBbhBqMCxbH$!WnXs*_-;DZl>DVT#V7im0Q` zpDNX++4pPrhKV~uukiajD)yRFV#u5RwmM%DEbLymGD|u)$(xi)bcT^U`<~wZAnnK2 zGeP^#L(^#gY+b~r>)0e<%E4WG@pTfu3l^g35RM|)Gn$=vEhnJQ-GoeH5UwkXthZgE zm{3NrC9#PCyu5_GM2{K{hKd91l zAD@0l>9IHL>htOkDsfU}G}+>vfC@ zM&xw89797gdd1&REh*kFR+Fa^8qkWjQ49jme4niP-dL*D^;#_~wdl#tFfNu~ikcA! zH(GC`W^T{vQVUFiE{M!$ywY;$2!S26v{;J~6ju|qIa{-sSPBI_tLn0n7@z^a9B4Q@rj)KW+sO^rB`tZoM89kt&0eiWl6bl^ zKdaQdU>|)Qn!p>x7~091I6mMLa#=K>k~kDO<~fXFuSsEwnQRujox;%xVxnZG{+Xbb zJ+Sfe;v$TarFO}nq3_s1V_Fi4l0RYATERDP9LPFN33LAJ6V90Uzun3E-~Ff)f|cLt z4?eekTR4cQq{z1Ok-*l+xWo1>7N;ML)qjTV)l89Qx~2QiEA~@=KSB+xUVn1Y(W^A7 zO>C*P$POd0w)QzQ;9RSHaOX(>-G6;UA8b7l|`ew1QI-Hshn_{2E*+3I?{Y`WFv z3B;!4?Jvsu_6rmrh455I{LI47N3i{TZa!gqPVw~ETi%r4TvptpYEBbtMw%CB@*W(% zgZgxaKfV9wGyK;r;&e__xyv?Bo+M8?SZxP^3jL-N?I|`$4Nh* z{PT4r80$Hvjn<7>d?#c${Efz2cpz3wuK#*zMcXzxorLe}1S}Qvbgh z3|Nj_xO`{(=T^Tq{z2Kq#zh zHj@!!hDNaM+QE+I<40fyRfD~M8jWL!ZCDuaf5C8e1YZPn6==Dj!1$$p9=W`8N5Rz6 zvpbxegFsD6P?eIHeUjtj&jF-I?!}0uO8Fkl$$Tk;UbOD&zeFiv?1PI*PtrTo%IkHKlWN#TVfqn+t8xpYNwbr*kkZ2eYy z;Chl0YX8{@^;9eP{;{ADCCb!1U7CC+b6hb1m5+!Gj zo={89N^o)YYHBjU)hKlH-)=8V_SuRpZiZ}raGl;0h0j{rBvYO6bN_Xm@IQ@Rg1KP`qKSepm6CaW`TK%z^soOt( zez2pX{%==}`u6`RE7N~E#1c>t>C(;Qyl*wPwSLEevGzT=f@XANG-wJ!O^{<`K?wKQ zs=>rGX_v9I%iqq~sD@pXEu3|$v9WFHa#-%X@oS4XS&%|eIli6zn|1E^b^qA^HPe6! z_neC6S;MX$b~{kG+~qO&5?Ff<@m9BZ=NgCe&+a;^&Recg0n09S)w+0sV}D+)!2#sZgSf=2NJFe=6lsSjG2}t`^22wS4j}b3zg&(gJs7D@Qe}ShbbBdZmtTdzpqCR1C<^nY%@op4pJ=OdgJ2SK?Z$|76+y9`S8xZW z|FP#dXm}S`dF;&biK9vZtpBH+D@FIlf5lE5bn_Y)0jD%YzEXS8k1?Fo5&A~b(NAJ_ zCt}*BCYD|36L=^I=nt|*<3H!^dNthH01C+aq@!MJ1J z?4xR>@L^#74t$MS&i7P4$ZoxCPV40v(Qgf3PG#$3`EYtlq63hISj}&l z>*J1^ZA|L7b*hnRFDQjxZ=Q1_xp|v?GexV0T?@l|of+f$kl3$U&}U(ivABh;ALu~% zA*5G7+$mVLT={sV-mlk{7=P|5u1D4IR6$W5+Y|K!KVdV|S+<`S(NfHr;)*o)8Fuha zsk>{iJASU@+;#~%tRU^j6eEo}55MvsIUTId5y{V-D;%fH!S`I|wux(D3wuq#d{H&oWBNLqP9 z0+#LN86F(o)OQkm>{n@yy~huYd+b+bf4&FjkMaBY2d*4@56VOd&=<~^9)<}40}!T3 z1v_ICQLgRjG(<1(sr{_$oGYy77x|NX9CuVOz4V!!qoC=Y(LDfE0@PN~YBULj0YLQ>~$&Zo5N)bH@< z4M#g4(#CRe5Qem5bxXpbfW}Py+9dr~Ut=^d;cmWY{iUp-xne=tR(_yX0#J-X5nm9y z#kzG7QQc$0GNar63N92ki=`8f&Gg4T|_#$2* zrUJR$>*a;3#L!x)Ih#&ZqWAS2(sdN+;S5*cmx7raisd3wIW zJbTKc$Y2g^Fp_Wy+j;A&W&;{=LSS5_D$AW`R(*kWFSlKZYdwNFh8VC5eOI#1@!R9GVB6mG*wB>( zcgbWOju4>DFbCclAO9*h7R@}+Y*XhyIX-!3Jjnjbb^A1tL;<7Ib2`Mh$EJl^9;He# z=bhL=MLZSr=!Wkz_R|BoJhp1NOV(;i2x}|fpsRa^rcXA&0nTFXZ-G^*@K*~WI|Hmx zf{8s$ARIPxX()h4lT%O7Pz@3jeNkzzpmG_!<|(>X@4~+)2`W&}CKLLLP@~v50d)1{ z?CGnj@2_}wK9j0b1%$shDZ;)Qd)T(bbScx zKtIl}fkSgj40t6@pEe&Zch@Az&$fOvkd1wyQ1|InYlZLEa{VmQ%JP;;2*=GkUk&0U zQ}1E9(IqOm9l;$0x}Y~mg(jJH5n#6*KX}`ncegT)RFoPU1;f?hvw<2@&$l65E;6V- zF|{vrm*R6K0XoVhZ*<|=kL}q!-34Q0BSbR|l2i)&_?T%_MbQjCQO%HO+W=%nAW(A& zW}p|{(UR}4SI$G!Av*imqo;$VLvV#(#105dP4TmIwi>W6r}{Lh>*gqo4u=S63Z0Ic zwYL#P5u7TZ-dF9KywCBq;2uT4{8dZCkRJ`B_!;?%3`_rsPI~J2%JNUh{oe)=C{O%s zcL)E3?~<99ThaFnXUofKJ8!eUUdxsf&+`lzH~&Z)HmhGhpY91?Z5t^qS_^TSIo*%F z7BH$zTsjxyl#`vSj1}%nE)h>;cqhP9SxnV2U|mbCAu1|7IqtFk-eWB}PNdx^;C|s? zUT;Kxw7`}zr*^=neEA`=Lxh(6xo(QYQd`)wT-d9U^eW>v*(>}ysM}Jxo|>Y=%mwL) zVZWL$MYbvL_xhlj94m8aUpoxc6E09CC$THjdO&2^vd6F)thg>uu=Lhk*80lKnM(o0 zm@CcJCnls~D8+#MP;wQL=r$S7s+gj2X!#sC^m7l69?8gn;r5wfB<#Oqh9}ArV<91M~1bg(_-*mZQN0_nT!NbmfQr zT-xLqN!9lm-?3A+3#(6iI+N83#4f8u`HVfm@N^qt#mt}qG8L`+7g8W)n!qb__ME

(YMsMa4BHVaKDhb*~XNcZesTt$jQ9| zgBY5oWv6>VrO>0VU2B)M-vBVnMDuf&JOT#%eJs$=MgwR!TJHXKo;#f9wG#_~QOY-% zmdUPWhOA~bk5f}0W4CS(j^kW~l9}tqLvz)6azli)*-vn<@-AdRTft;y)nN1AG<3S8!K6`=v`$XYYaP(uHCvGi)bi#1IkzE6+X0-h%NoL zulP_{$o5{DP@m9F?uglSO?b-oqRNtc<5DC;tnQ&N`wDSCnCPkJqjL#V4lZ&9dwdwJ zSVWTvn(^LOrINvdv@@0Sc4xlHyb{$T#fnRO5D)}Pg%iWW!$3FKjrNLLr2K{U*%{uc zdzlQJ-(H@OjB!`TwBY0zY(vhT6il(WZzyV=O#L8c4k9X3&{r*Yw*&ZaTeVQa!v9(-AoJqc%awUxM zna=CVhc(+1;r5B?`fML~HUw@md!83qP%r>&e5m*=Ss_%hRAdFWRg|2$+C>&`J@-^X zy)Mlz=kdQ0_7ZcA79(atYEvKfGw>tF~2IX-OhKjYvt* zD~%BO%;&ZWzvk%W*E~)|VLG@~ciHf;h|{%_8IH^kBBzvNC$xeQIUo{`2)1AIYG43$ z;#&fr#-5zLJv-;8BOU_H%<@_7eYcqd#qk}zQ%^s>+BdxhNY5UC`3THuHL`G+z22^J z)XOp|NMIq!-CikW4PA^yYx0^I7{waRhQRcNbf|gNAkpFG?sw9`-_nr|WQ})BBA&av z!y!uf4Ibp^;7)pV7CtsxwPoT>Kar>Fw4IGiMY1v6@!2$qH69t2`BkIEP-C#3&rdB!>3og^93uY*6(Lc>oV#0m>GwAIs@N?y z{&cL0-1A%&OYutGT6PqpzB-q0U%Y^ITZ%)c26swaF*-G8ObX=VM)vecY&4eK(sCp)Rg|g$0@>;DPwP$X*sKBGtDFhVEUq>#v7b!3~|(=3(zB4xNwp! z3wY>5B8u`i@nOQ_4tQpy!ARIPr>>BUi-7AxfI z;u4X(`9)k6K=%(b=e3!Tc|K;lmyYC%T~vDtF%R}1rxHU!fdF>3|rTjZ1h6}ndE zC{!Zc4>l!cqY|G*RP@d<50IbUI0`^Iy#A)hFoh=y+ZSJAK4kE5F;suJBhN~9Q4TTi z`TdA9zk`V^SqMb&>^AkEY1!j_KoiG z)Yz)XOl~xd(&?=w=UN31Q8%rROt3%iTjED6;6jZiEQKPu=jfgoh@%7{VV*GpVhT!3 zmzrN8e0=Poc_KFJX&)DS=)R%)p%H*&$8@C51@!qdZK;MvEfc)y4zb0w=S3&EV<^l5 zS>@9oJzOs)=~O?URQdIch`5hTcj}o3B2CmmMOO>&3p4p6v1|!xZ{O`e>kl)c8y+d6&8Glqt{Z*7+dqk zo=k3m+1-`2=mr69TrdI(Lzm?q-)t$-Y$^XSLt<+u3nYRqmL6KbT@XjWJiqh1mL0U8d#&8Ob9c;!A#`0tim9BP z3xMGynI8p?P6E>mx(#gP#)K&)@uj@Pd&kF`#RmAvx3?_&@E?s_69CqsIeHjnVP!v( zj|<(LARm3Mcz}pK#86j2Lv6({f&EkCgV)0_VGIAw+7!FW2wVz|4#~LW={)>MO2<_8 zWt4Of+r#mt__|mx)%TyEf!Ul=25}Neq~>=f$QUUrPn=QG$#9Tbfl75$&TIj?xK@q7 zGq2R?0P!)*(}>to{TSu)oDBPts@U&J-B~_3UT|Xd^%~fGXz*#KcFF*CgBE|6dJ?sZ z;cUE?K7V{rpIYX$Z@}3`8AlNa3zdls@4jlBkCbtV0T+mCaG8!NNO)-~ECik#<$*5RH;|-tv}QV4K#b%8IpIFn+sOpW=-o=ok;L zwe`t9lN>vie!DfS|5nFE#>|rqLCUGjM3W$ZW?l{_wo!CdL}BUzS~$ub2C{xtQKVU~ z56gz8f)I+{4&OXJLUht2mzJ9rjBKm4Q$W_~HY<;c3xs#vS2 zppCZvmzvP41w?+b1)&_=lef%_hNl5xrQ`|2Cu}TCW^`bh;3d6Do_Yi;-?=D{+>O;y zIu)6iUK^urehh9DH=D?LCk(L60W(UR7-+dkvjWOnS?(e`zkYNQ2Ts-GoOK#bZhe z3;f*^0N#*Ye|prwI3$~udPyz2pUE#dA?O-5b))U6XrwGuaUz!aUjD9TasmFXP!?tO z8(;sB$?m0CL#cR2Ay^bE1}NOwe&+R*3;6?aIB(P`--V;JUwsR~QE28+@qO8XFL&#{ z!EuzSn}D$T^q?a{Z{vD>=8qWqPi*l=pz!!Sr$GG2x~Ni~`dfli`c&q2lY861N+4KA zw4-T%Y#sSbKOhQ>l>^xbTKUZcK0NGLSU!c;)XrThC#!A??zk}8;3K@D~rz$RMD0kwzdbR zyhc_M%9-?2>|(b>&U07Qj76!Wgee0of`LUvsTHAbN)fB&gXzu>RBgva3$!rC3-V%^ zo@_(Lxv)q22Hhv~=#WL+`_w2na5@1q6ZodfV~lY&*C9Z+qGZGwKI$ED-{WJFU>wtc z2Gp%0m(?G^uRBW?b8p}%SE8@&GZA3UMcxJ!P9fIMyR4B-IwdZQLn@Z<_$tQ zX?P10zg!K#eoAAC_0kCca^?%@gC~!=y_#4yj=q5$($weB%=M|*%~7>VUnSNlrt1AC zQ)X?#)Z}Cj(QV;G%Yn;o`vcECw-FgSa$CZ+g<&E5lTxcT%sZ&tNcRw9M`Eg&mDdNi z#2g7(dx+k+0O#h(cwuiazlj%Fx4--bId;01ChVm4NVirX0b4Q1fkx>R!5~9O zo<_!5x|jEu3H?iC4g^(KJAG%0kb9x;oJT^vW~vfSHAzA+`GUNr!IX>!=!}kBRb@QS z8GQ04JX_?F*;W&4RpGZ>G4_YpE_Ah8i$0=4ApF*9XnY%X%N(}BCU}p78}7~hZm^-1QE#g#ERh?{c@7_vR&(~~QsaT=s%>t&d?P`-;NGw36^ntqwI=|PlqCYndgmd*8h8h^e zbJz9-N4~UL>-U4oi7c3MDHPpvsrIe%1f+8$O-e^F7zZsqWm4-Ek*g8(+`nHWEGD z|9bBKnBBh&CjVHqk8l6YIsV5945tm1N;i1Nw3tU2hh$K;9q@0GgE=3vl|9U*GV;bP z#rSOq(z(NQaVF?fw$|~9q$Slcmb1rzwUfA$;A7xs(Eq4YCw?}Y4$aS;)%J3K&^0OH zGqjF2Whm&HR`t*tAPL1+79v59?kQ{7w2B&U+~Bdz@l2ouLV|kH1%>8Cr4o#Njk-z`I`D=u(_C6i? zg4crND?$CN;0co@e=&voEPqRy`g?%$_)7?83m|7}4y`1>xh5vhu9vH(IvEv|CgUDl zwP~}$NC;d_?3+ZsX=5EXMc;_m;WEr6h`2K$kKxa!DacG%D(I-pvP3O`-H+yf)Txu> z2cUUHzCFiTso!xS80qF~yOc5-Rojwkbw!8%Xea$;dbN~nXq3T&@P@Ov%RE4>ej9f@O zW7P&(OV0YDRs`V;Jb~FAAWywiOFz!v6Vkvs=DLw*^r#f;6qr6M#wl`_!aeu}g85^7 z{xyR6{J(sYt3+xh@5G_C=^?-RSUE4s(EI7yh3rRTxTUdzQ}(HsVhxeXi1PV$yZk;0 zu|X=i%K!669z$B6Q@+QsoG*8lmO4opfazh zZKK<1U%%c_&ols?yksL_n}53{0hX;0i(Yn&c}_>KLM`ebPtujs`4hu zlJUW?+U09SvS=p_xS7UM(t<~9LlQZ1z?zi*X;HlO=8MQSY88Jm-u~j8_G4J#3Ggow z$R9)YuffhM4EKKpcK#^>KS9RhVCR4PqX0kc`3w1(qa!!3j|>~!geYxP5iqcA0f*M& zNr}+m6Ab&mJ1>r@EbDR9?lwlk#bfvnxD!b(7bv+<|6iE_KV~CGwbQuqX>=JV!U>lXovGaO?Ty=c4jxBunOd7`|~?t)-$^v2z5GDqNB@@$w!H zg!Hi2HtzXL7EyRza1P7x znE#fLFPB_@#?#y}jn1LT-cw^~_h=bO*9BgCQk@Izu8^p=HxiP7(DZ^_Jj8KLHg2{n z{k=x%CoQ!P%qZEi`yE)`bh_FF+GW&Ur_rZB_3__inBM<&67+wd=clBQETPl*3Y}IW zTY*`=KCXu!=fVBe7u_7g=F54}i+II4mp#xo*V;G7e3XxO^0qZt649}nv?4W;M4P-b zRzpZhuR?r$JNkH&e;1lBHfyNBFi$2sKSz5uHl>GA@A>NS2pxrR}w=);5oK2!qcZO$3gXwSDm$5obnA zezYYpxJHjw_moLNVA|5TJWz}ZKEI_UVe9!46>GQlSUukqEdW}6Wo?3YqkKKIW<{?5 z!g3U2p2Uc~xPo&Qi`L`<*SOLI!d_&8PA!`wpoOVhF3=YAu-iITEf_Ks`^O@J(H>uW>LgDX5RiRk-sh3R&G3`VCXavWzWn<>oVBqk!rOv^; z>>}Ay4auLX3G~-DlMQrad3@Tq`jyvH{3rhupI@he{$NTUuSRy<5c^4VQdFpr>inYC zQYrAwqy*mBYGu=&NYEm#48kH?!8EmWzJX9oY9xQBaOqtJd0yu;#r*{F@=39@Cgz3p670_X^i43{JxyM zPE6-po0CX;Fh@NQi7~!oV-%xh5`%$bUrnZ_hH3@+N858}U`zugSMy-Fqnez7xN}x# zZnib}c!+dMYPScF?nt6EP0*3laM&dlh}ctDKqVEu>n|>1SUJtQ4r_iQA|>yd=et1 z{7GB6Nky)?XQOTm4+E32T+(>0EmAY-a|U!uPY=3+CU z@a-#)$ktupF6#73`wp~=!gHs~?SJ$vmbX6(=Qp zbev+5UQdG5EZw$H5bG)Mv|#3H56n$|x-?Yo!L@9@9J5L5tClVSB)$GqGAQnm?t$>5 zw|sm1GaVEX;xX3S5BuTg_E&f4udKI!zv?LhDgn(ov;JFZ1tzCf6p^RXst5I=iSK3J zw`-4Ja4;OZ=aMpeXJw%1T*sLE)A`_zv&;N64{-xGz>0Qk0boprt)ak^&;rgofku={ zeo6YFxkZ8iWlc)?&?JtS5PL;DO|5s&Did8}D_H?L@Su#BfpfT8s7uNmryfbTkvQA; zm6qBO$(*W`;;IC>G>BUHt$G}oI2N5mKZ@m5!7t%ToSJAp6!$=meBun*rNFu0sc(xU ze5k{Oi~tma+n4vfp?WSd974}C&g(e!Q8)638MBC7QB$$a=1hKEP!EHH&&muTnIz>! z373CwQ1Ko-n{5^X%E5lg&CAZ6z1SXB71c%U2bk5;)556@byQ*Eo*W;k@szsSe-8SlY)H zy)1JRdGOY+jsJrP>YqPw%MH5H zS|tSbflt)fxkGB}W|<56XAGI1>d4y zx3+4xWg-?4PqgAt38+bQpQ5biv2iAhuWVHwW23rOY-m=Ry?0qwWg5J3F&&XI zW0pxyhY!2ZOSCG1#%6N(BbEJ%`(Jslr~uX{64#fBMpCti&;o`FJXSPd!7CVXZ`2TL zzN&$dMSb+4dYf6!!im8wMOl&cH`}_V>CkQxgdePZcHgSs%nUF`v{crvdXvD~ z3rN3@zzYO+bbb(7)TBY~?O%@e5_Ny;Vyf=}r8EF+Nl5ut9>?98Kkj0N2nJ}s)Wr}4 za1hEICq9G$tO}!lxgrziK7%aKl;q$=ZS_BU_$~@svauW%L#DSCF zsnullVKl5z>|T&O<10G+1B+OLF)?FD1DYPf=Sce6RaWz|&!^)oaFj~^S$!MlVsl*> z^A0P<-gr_@_5$fU%k(N z4^sxN*CWTTAzP~?#|d`e3!M-@1|5|bdZATx+&5p{+o%{jH&EUa6&>6iZTkGI?E2LZ zPm77;ER*6V_=VZ_uGG zZ@y8g5cah8m0%M^o-$_xaiBJ|_(UKklY39gQZt{huL9u%>C=9Xc}^qSXykDLw_tD5 zVe-cG&^oKg{X#;wGi2OinJXg4D>hO$Ygk~;Rb^?|GIovF5o|XK9IwFjZIA=w%4r;% zzF>!KyN(gI{B!O)wg>?@Z5(4DrWl^SbZ*AeCv9;I_GP&d3ubM*2p>*TfBFw9Zo z7umXnDR#5GD|v=?gPuL{?XUyuZ0Dvl1G?PGQTAn8+L+{wbPp<-3){j$6{=sN>`I+^ zrmU>w*cU@Qhj2;0beK+Ddq|BlN-Yr0Qo~f_T{+vzgK}|+p(C<<6r1u$Sa&s%NwTdD z-6+kKC08t1_GVG~AiZAnpMtv?NJ#g$n&+x%jO%;g`$0?ZPBgPf&S68xLZ{nDn$H>E zK65J>o&azcEI2KWHnwM@(^AO=(dA5^uL#kcq14?WhKI<f=n}qybbo%^XYSR4G zo%r+alPpB)YA3U5-g&aiAj(ik0=5H|pgXM~IIj_1M|+k;w&ODBR#H&nQn(j1#z!}? zn=0DV@g##bM__hHUq9!KwVf;;1mls6-fx2Wm5FLz7c>Qmi?7(q@>4xU-g{@l$LSzo znO3N17ze?Nx)rS0gjodU$Kv9*u{u(Zuz9WWu_LjRo~FHwAp-*{?rY%34{^mtgtW%g z)7=%o3&Z!Zruz&7cBjFLRY#nWHA zK9%a!trn-`Ij``tUPvo5_{t1iY!+AYBI><_27aY0n}mR|iFuXAbBjVkQVyRJUHk%E z#>+V$xE382gG)5Q>RGEdvqj_c^jX8B#}c*HT8QUbc6x$6oT%o3VoZG2cizW7E)NBy ze!e!pa+u`n<&mb8uAc**QUZzO&BTT@GdpVlW9O`Wa#hsSC@F+eG1l^BH#>Y(!$JUu zEKDQ&;#zeC!|Le-NraK9&RoO=aS8s3=}!B>W=2hsi*fP63By^xrER8l`;8eBI;|JY zQZ|H0P0_1JazdDzKkl=rRmw#_Y_G<^oF0#N3=@~L>R7*yp2{eZP1|U|_>6gzv{F^@ z?N{xFSXV6gfdE>t`oU5xgopEW!gVSgJW%gkZq{IpwdPv9Rq4Bb6P;fN(0`<&|0=lt zAHFA*3xH~LOXivtypFkTXdU9eZZutK@i=={hSA|H*eH5Ovk+}f$-_{f9uUH3_g5si zblnOR@BX4TGp5wjIfT}IX;JC$J0pjw-UoLQ0?}g>jPtP)%kgT;nMn;bGB1 zOXsMLW}7RAe}`6uIlN!e)!o=s4P}kS0=J#SH zZuR+zFI#R84poLd6pL?LRBRr`h-)XEB*H=V!2+II6&{n156sc)hH$c>cqr?GaLH}~nR%(C`i&E>buYRmo3a;fEXq!}fQkOmdR zT|+qb+3j%D3N|+q`_e^?PbEpks>CP5YW#|+WIpp4xGIUPLS91NNpFgfqYR|wvR_p$+lko^C7x^Msjz1$`u@x`u(4^d)y4RS7BfCO^ zSIox^;^%BJ4`woW*yrHld_Ex2>$%y6A0H9!Jo|$xEQ&(vKNJ-C?nEb-Np0&}9_k#i ztXIJ`=leF8SBf);ru@F8uS~GE^=)3ct_FivVGT-hZAPbD0G6>PIi*RP?6HD+3n!=E zG7xUtS=5zcf)C>b;@iCAA*`>&*!Vcsr`+NN&Mowtd1-uoNabpycz#hn{g#^In+jW8 z@2X2-74NP`vMG17P7ZY| zAOX>~(I5dXW@qXP0x2$^$dW^Y1c?|SK&h)q2DHbwgg4tzT@Kgy$jEhD=Oeo$RLhqu z2sey^)#My>9NH?p`!_N9E3@J+F;_}*(eXTOmE+LFKakkyhaHN}R$4g#Bcy;^mA>O) z=W1xixG^gvHMFu&RKqv%Rm@E$khM zV{NFdnTh-2Ez>S77gXh4dBkc`pvA?BDyPe*MiqV8!0@c<=<4lPUpm*FTyyV6(dBmA z$(=^ewm-h|$07k8ludQgw?1d=HuF*m(w623*Dn@YJ_y?n~tXiZJmgt%$Q!*KokNO@1UHkfil_!Rf0;*e`4eD_H$`Kew3Tn zt~aNy?)P7D>E>joyjjor#j+>LRBjKnQkhb{L-4HHB(H8)v1RTH7BFyz2kz;9oIJJW zcGhPDqzNCOW2Pc;*^yL%_FExwkyH(3O&ovz&R_lYyXKm^zui(+H_i526Re%?J6-5r zYN!5zS!YhXEDT)fYvb{7@s>1?LbVjPs88Jt44nU*kM!L)4`uthBxUuwGi#&2{OZ-6 zX`=RWUBSn+or1?_mkKZPbl=@5b1740#g@RVlqw$Npc|ecNo4BaS0N+7d4tuHZ&W&6 zD7rK|Zq-Hm&#h)F9?(##NJU1ILo5%~_lc+_Jd1QvBF) zt)JVjK1+CIKCvdJX#MQmxHQndL?kXBLRBJYpCf`h95p5GdVE`JYHtpAw#ujx;%@?;s zH_ca?9@u+m?LIl}w=1_t-*;S^dC%!e)RL2~>p&AT4yd~>ry^||bw-+d1?|E`nzW_H z22W(GTrQ=ZpC(=H8KiME{$wn_((x51P3QUU|O7YJcKZ+vWcmJ{JB3-UIqABvXD4Gm>*@xi1#96&j~iQh(bMZ(CfunRH=`(&?mG zM>>m6EWM)QS9|=)i6Wo&n2D-+Hm|OIKDXw|))ikPdm9)SDz&c63ay)J)*8FaWyR$z z!#kZuGTVYa3q=&FO_wU%DhRxD#^+Kd@Va$lWw{N&nR36I$w!xq*7`257C>TyH~(&V zJIyAlF#O7`@{N2*Z15J}k)cg+|E+6tS0@K7x^*pdv)A?)8@B(Fn(N(i$74phf7w4|-ySZIoO|v9h?P053YTo(Pfl)4< m!Y@wu-3qlcR(UmL#wCB3{0!fDz%zUA96+x$2=m$h-vj_6BB;y& literal 0 HcmV?d00001 -- Gitee