From ad76d76b4c847b3f40b97c29ad8c5278ee0b7106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?TD=E5=8C=97=E5=B2=B8=E8=8A=B1=E5=9B=AD?= Date: Thu, 27 Nov 2025 15:28:20 +0800 Subject: [PATCH 1/3] optimization function of deployHousekeeper --- cmd/deploy.go | 21 +--- cmd/deploy_test.go | 2 +- cmd/upgrade.go | 9 +- pkg/kubeclient/ctl.go | 192 ------------------------------------- pkg/kubeclient/ctl_test.go | 32 ------- 5 files changed, 7 insertions(+), 249 deletions(-) diff --git a/cmd/deploy.go b/cmd/deploy.go index 22b4cc74..4b5a2087 100755 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -281,7 +281,7 @@ func clusterCreatePost(conf *asset.ClusterAsset) error { if conf.Housekeeper.DeployHousekeeper { logrus.Info("Starting deployment of Housekeeper...") - if err := deployHousekeeper(conf.Housekeeper, conf.Kubernetes.AdminKubeConfig); err != nil { + if err := deployHousekeeper(conf.Housekeeper); err != nil { logrus.Errorf("Failed to deploy operator: %v", err) return err } @@ -358,7 +358,7 @@ func waitForPodsReady(client *kubernetes.Clientset) error { return nil } -func deployHousekeeper(tmplData interface{}, kubeconfig string) error { +func deployHousekeeper(tmplData interface{}) error { dir, err := data.Assets.Open("housekeeper") if err != nil { return err @@ -371,22 +371,7 @@ func deployHousekeeper(tmplData interface{}, kubeconfig string) error { for _, childInfo := range child { filePath := filepath.Join("housekeeper", childInfo.Name()) data, err := utils.FetchAndUnmarshalUrl(filePath, tmplData) - - switch childInfo.Name() { - case "1housekeeper.io_updates.yaml": - err = kubeclient.DeployCRD(string(data), kubeconfig) - case "2namespace.yaml": - err = kubeclient.DeployNamespace(string(data), kubeconfig) - case "3role.yaml": - err = kubeclient.DeployClusterRole(string(data), kubeconfig) - case "4role_binding.yaml": - err = kubeclient.DeployClusterRoleBinding(string(data), kubeconfig) - case "5deployment.yaml.template": - err = kubeclient.DeployDeployment(string(data), kubeconfig, housekeeperNS) - case "6daemonset.yaml.template": - err = kubeclient.DeployDaemonSet(string(data), kubeconfig, housekeeperNS) - } - + err = kubeclient.ApplyYAML(data) if err != nil { return err } diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index 4d4714c5..3dd8c3cc 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -131,7 +131,7 @@ func TestDeploy(t *testing.T) { }) t.Run("deployHousekeeper Fail", func(t *testing.T) { - err := deployHousekeeper(nil, "./test.yaml") + err := deployHousekeeper(nil) if err == nil { t.Log("Expected error, got nil") } diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 6c9ced92..9b95a0f0 100755 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -18,15 +18,13 @@ package cmd import ( "errors" "fmt" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" "nestos-kubernetes-deployer/cmd/command" "nestos-kubernetes-deployer/cmd/command/opts" "nestos-kubernetes-deployer/pkg/configmanager" "nestos-kubernetes-deployer/pkg/configmanager/asset" "nestos-kubernetes-deployer/pkg/kubeclient" - "path/filepath" - - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" ) func NewUpgradeCommand() *cobra.Command { @@ -92,8 +90,7 @@ spec: maxUnavailable: %d `, clusterConfig.Housekeeper.OSImageURL, clusterConfig.Housekeeper.KubeVersion, clusterConfig.Housekeeper.EvictPodForce, clusterConfig.Housekeeper.MaxUnavailable) - adminconfig := filepath.Join(configmanager.GetPersistDir(), clusterConfig.ClusterID, "admin.config") - if err := kubeclient.ApplyHousekeeperCR(yamlData, adminconfig); err != nil { + if err := kubeclient.ApplyYAML([]byte(yamlData)); err != nil { logrus.Errorf("Failed to deploy Custom Resource: %v", err) return err } diff --git a/pkg/kubeclient/ctl.go b/pkg/kubeclient/ctl.go index fa714824..35bbfeea 100644 --- a/pkg/kubeclient/ctl.go +++ b/pkg/kubeclient/ctl.go @@ -23,13 +23,9 @@ import ( "io" "github.com/sirupsen/logrus" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" apiyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/discovery" @@ -44,27 +40,6 @@ import ( "nestos-kubernetes-deployer/pkg/utils" ) -const ( - // Constants for CRD API groups, versions, and resources - CRDAPIGroup = "apiextensions.k8s.io" - CRDAPIVersion = "v1" - CRDResource = "customresourcedefinitions" - - // custom resource - HousekeeperAPIGroup = "housekeeper.io" - HousekeeperAPIVersion = "v1alpha1" - HousekeeperResource = "updates" - - // NAMESPACE - NSResource = "namespaces" - NSAPIVersion = "v1" - - // RBAC - RBACAPIGroup = "rbac.authorization.k8s.io" - RBACAPIVersion = "v1" - ClusterRolesResource = "clusterroles" - ClusterRoleBindingsResource = "clusterrolebindings" -) // CreateClient creates a Kubernetes clientset. // Parameters: @@ -122,173 +97,6 @@ func parseYAMLToUnstructured(yamlContent string) (*unstructured.Unstructured, er return unstructuredObj, nil } -// deployResource deploys a resource using dynamic client -func deployResource(yamlContent, kubeconfig string, apiGroup, apiVersion, resource string) error { - client, err := CreateDynamicClient(kubeconfig) - if err != nil { - return err - } - - unstructuredObj, err := parseYAMLToUnstructured(yamlContent) - if err != nil { - return err - } - - _, err = client.Resource(schema.GroupVersionResource{ - Group: apiGroup, - Version: apiVersion, - Resource: resource, - }).Create(context.TODO(), unstructuredObj, metav1.CreateOptions{}) - if err != nil { - logrus.Errorf("Error creating resource %s: %v", resource, err) - return err - } - - return nil -} - -// DeployCRD deploys a CustomResourceDefinition. -func DeployCRD(yamlContent string, kubeconfig string) error { - return deployResource(yamlContent, kubeconfig, CRDAPIGroup, CRDAPIVersion, CRDResource) -} - -// DeployNamespace deploys a Namespace. -func DeployNamespace(yamlContent string, kubeconfig string) error { - return deployResource(yamlContent, kubeconfig, "", NSAPIVersion, NSResource) -} - -// DeployClusterRole deploys a ClusterRole. -func DeployClusterRole(yamlContent string, kubeconfig string) error { - return deployResource(yamlContent, kubeconfig, RBACAPIGroup, RBACAPIVersion, ClusterRolesResource) -} - -// DeployClusterRoleBinding deploys a ClusterRoleBinding. -func DeployClusterRoleBinding(yamlContent string, kubeconfig string) error { - return deployResource(yamlContent, kubeconfig, RBACAPIGroup, RBACAPIVersion, ClusterRoleBindingsResource) -} - -// DeployDeployment deploys a Deployment. -func DeployDeployment(yamlContent string, kubeconfig string, namespace string) error { - clientset, err := CreateClient(kubeconfig) - if err != nil { - return err - } - - unstructuredObj, err := parseYAMLToUnstructured(yamlContent) - if err != nil { - return err - } - - deployment := &appsv1.Deployment{} - err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, deployment) - if err != nil { - logrus.Errorf("error converting Unstructured to deployment: %v", err) - return err - } - - // Create the Deployment using the Kubernetes clientset - _, err = clientset.AppsV1().Deployments(namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}) - if err != nil { - logrus.Errorf("error creating Deployment: %v", err) - return err - } - - return nil -} - -// DeployDaemonSet deploys a DaemonSet. -func DeployDaemonSet(yamlContent string, kubeconfig string, namespace string) error { - clientset, err := CreateClient(kubeconfig) - if err != nil { - return err - } - - unstructuredObj, err := parseYAMLToUnstructured(yamlContent) - if err != nil { - return err - } - - daemonSet := &appsv1.DaemonSet{} - err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, daemonSet) - if err != nil { - logrus.Errorf("error converting Unstructured to daemonset: %v", err) - return err - } - - // Create the DaemonSet using the Kubernetes clientset - _, err = clientset.AppsV1().DaemonSets(namespace).Create(context.TODO(), daemonSet, metav1.CreateOptions{}) - if err != nil { - logrus.Errorf("error creating DaemonSet: %v", err) - return err - } - - return nil -} - -func ApplyHousekeeperCR(yamlContent string, kubeconfig string) error { - // Create a dynamic client for interacting with the Kubernetes API server - client, err := CreateDynamicClient(kubeconfig) - if err != nil { - return err - } - - // Parse the YAML content into an Unstructured object - unstructuredObj, err := parseYAMLToUnstructured(yamlContent) - if err != nil { - return err - } - - // Try to get the existing custom resource - existingObj, err := client. - Resource(schema.GroupVersionResource{ - Group: HousekeeperAPIGroup, - Version: HousekeeperAPIVersion, - Resource: HousekeeperResource, - }). - Namespace(unstructuredObj.GetNamespace()). - Get(context.TODO(), unstructuredObj.GetName(), metav1.GetOptions{}) - - if err != nil { - if errors.IsNotFound(err) { - // Custom resource doesn't exist, create it - _, err = client. - Resource(schema.GroupVersionResource{ - Group: HousekeeperAPIGroup, - Version: HousekeeperAPIVersion, - Resource: HousekeeperResource, - }). - Namespace(unstructuredObj.GetNamespace()). - Create(context.TODO(), unstructuredObj, metav1.CreateOptions{}) - if err != nil { - logrus.Errorf("Error creating custom resource: %v", err) - return err - } - return nil - } - - // Error other than "not found" occurred - logrus.Errorf("Error checking custom resource existence: %v", err) - return err - } - - // Custom resource already exists, update it with new configuration - unstructuredObj.SetResourceVersion(existingObj.GetResourceVersion()) - _, err = client. - Resource(schema.GroupVersionResource{ - Group: HousekeeperAPIGroup, - Version: HousekeeperAPIVersion, - Resource: HousekeeperResource, - }). - Namespace(unstructuredObj.GetNamespace()). - Update(context.TODO(), unstructuredObj, metav1.UpdateOptions{}) - if err != nil { - logrus.Errorf("Error updating custom resource: %v", err) - return err - } - - return nil -} - func ApplyYAML(yamlContent []byte) error { var config *rest.Config diff --git a/pkg/kubeclient/ctl_test.go b/pkg/kubeclient/ctl_test.go index fce3e8e7..f99fb0db 100644 --- a/pkg/kubeclient/ctl_test.go +++ b/pkg/kubeclient/ctl_test.go @@ -130,37 +130,5 @@ func TestCreateClient(t *testing.T) { log.Println("TestCreateDynamicClient success") }) - t.Run("DeployCRD", func(t *testing.T) { - DeployCRD(yamlContent, kubeconfigPath) - }) - t.Run("DeployNamespace", func(t *testing.T) { - DeployNamespace(yamlContent, kubeconfigPath) - }) - - t.Run("DeployClusterRoleBinding", func(t *testing.T) { - DeployClusterRoleBinding(yamlContent, kubeconfigPath) - }) - - //t.Run("DeployDeployment", func(t *testing.T) { - // //patches := gomonkey.ApplyMethod(reflect.TypeOf(&appsv1.Deployment{}), "Create", func(_ *appsv1.Deployment, ctx context.Context, deployment *appsv1.Deployment, opts metav1.CreateOptions) (*appsv1.Deployment, error) { - // // return &appsv1.Deployment{ - // // ObjectMeta: metav1.ObjectMeta{ - // // Name: deployment.Name, - // // }, - // // }, nil - // //}) - // //defer patches.Reset() - // d := appsv1.Deployment{} - // t.Log(d) - // DeployDeployment(yamlContent, kubeconfigPath, namespace) - //}) - // - //t.Run("DeployDaemonSet", func(t *testing.T) { - // DeployDaemonSet(yamlContent, kubeconfigPath, namespace) - //}) - t.Run("ApplyHousekeeperCR", func(t *testing.T) { - ApplyHousekeeperCR(yamlContent, kubeconfigPath) - }) - } -- Gitee From 33cb76dba20640774bcda1130397887312483391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?TD=E5=8C=97=E5=B2=B8=E8=8A=B1=E5=9B=AD?= Date: Thu, 27 Nov 2025 15:30:20 +0800 Subject: [PATCH 2/3] code specification revision --- cmd/upgrade.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 9b95a0f0..30417965 100755 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -18,8 +18,10 @@ package cmd import ( "errors" "fmt" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "nestos-kubernetes-deployer/cmd/command" "nestos-kubernetes-deployer/cmd/command/opts" "nestos-kubernetes-deployer/pkg/configmanager" -- Gitee From 609898b09c7fdd8fb61f6c6d2be7af85f8ad0b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?TD=E5=8C=97=E5=B2=B8=E8=8A=B1=E5=9B=AD?= Date: Fri, 28 Nov 2025 15:42:45 +0800 Subject: [PATCH 3/3] before applying the network Plugin, wait for critical API Groups to become available --- cmd/deploy.go | 57 +++++++++++++++++++++++++++++++++++++++++++ pkg/kubeclient/ctl.go | 4 +++ 2 files changed, 61 insertions(+) diff --git a/cmd/deploy.go b/cmd/deploy.go index 4b5a2087..17b02c2d 100755 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/discovery" "nestos-kubernetes-deployer/cmd/command" "nestos-kubernetes-deployer/cmd/command/opts" @@ -272,6 +273,11 @@ func clusterCreatePost(conf *asset.ClusterAsset) error { return err } + if err := waitForCoreAPIsReady(kubeClient.Discovery(), 60*time.Second); err != nil { + logrus.Warnf("APIs not ready in time, proceeding with partial discovery: %v", err) + return err + } + // Apply network plugin if err := applyNetworkPlugin(conf.Network.Plugin, conf.IsNestOS); err != nil { logrus.Errorf("Failed to apply network plugin: %v", err) @@ -358,6 +364,57 @@ func waitForPodsReady(client *kubernetes.Clientset) error { return nil } +func waitForCoreAPIsReady(dc discovery.DiscoveryInterface, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + requiredGroups := []string{"apps", "apiextensions.k8s.io", "policy", "rbac.authorization.k8s.io"} + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for core API groups to be ready") + case <-ticker.C: + serverGroups, err := dc.ServerGroups() + if err != nil { + logrus.Warnf("Failed to get server groups, retrying: %v", err) + continue + } + + found := make(map[string]bool) + for _, g := range serverGroups.Groups { + found[g.Name] = true + } + + allReady := true + for _, rg := range requiredGroups { + if !found[rg] { + allReady = false + break + } + } + + if allReady { + logrus.Info("All required API groups are ready") + return nil + } + + logrus.Infof("Waiting for API groups: %v (current: %v)", requiredGroups, getGroupNames(serverGroups)) + } + } +} + +func getGroupNames(sg *metav1.APIGroupList) []string { + names := make([]string, len(sg.Groups)) + for i, g := range sg.Groups { + names[i] = g.Name + } + return names +} + func deployHousekeeper(tmplData interface{}) error { dir, err := data.Assets.Open("housekeeper") if err != nil { diff --git a/pkg/kubeclient/ctl.go b/pkg/kubeclient/ctl.go index 35bbfeea..51dc9460 100644 --- a/pkg/kubeclient/ctl.go +++ b/pkg/kubeclient/ctl.go @@ -123,6 +123,10 @@ func ApplyYAML(yamlContent []byte) error { return err } + if unstructuredobj.Object == nil || len(unstructuredobj.Object) == 0 { + continue + } + if err := applySingleResources(kubeconfig, config, unstructuredobj); err != nil { gvk := unstructuredobj.GetObjectKind().GroupVersionKind() logrus.Errorf("failed to apply %s %s/%s: %w", gvk.Kind, unstructuredobj.GetNamespace(), unstructuredobj.GetName(), err) -- Gitee