From 29c63fc8248fe68d1e7e803e3ba051626fb3df11 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Wed, 21 Aug 2024 20:53:14 +0800 Subject: [PATCH] KubeOS:sync code from upstream master branch Signed-off-by: liyuanr --- 0003-docs-update-readme-and-format-yaml.patch | 742 ++++++++++ 0004-KubeOS-add-nodeselector-to-OS.patch | 1230 +++++++++++++++++ ...transform-log-timestamp-to-human-rea.patch | 33 + ...unnecessary-fmt-and-add-printing-for.patch | 137 ++ ...-support-setting-TimeWindow-and-Time.patch | 628 +++++++++ ...feat-os-proxy-add-ExcutionMode-to-os.patch | 865 ++++++++++++ ...roblem-that-proxy-will-get-all-os-fo.patch | 628 +++++++++ KubeOS.spec | 15 +- 8 files changed, 4277 insertions(+), 1 deletion(-) create mode 100644 0003-docs-update-readme-and-format-yaml.patch create mode 100644 0004-KubeOS-add-nodeselector-to-OS.patch create mode 100644 0005-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch create mode 100644 0006-operator-delete-unnecessary-fmt-and-add-printing-for.patch create mode 100644 0007-feat-os-operator-support-setting-TimeWindow-and-Time.patch create mode 100644 0008-feat-os-proxy-add-ExcutionMode-to-os.patch create mode 100644 0009-bugfix-fix-the-problem-that-proxy-will-get-all-os-fo.patch diff --git a/0003-docs-update-readme-and-format-yaml.patch b/0003-docs-update-readme-and-format-yaml.patch new file mode 100644 index 0000000..7430f07 --- /dev/null +++ b/0003-docs-update-readme-and-format-yaml.patch @@ -0,0 +1,742 @@ +From 1559b84824331e4c7978b0777d274119cacd82d7 Mon Sep 17 00:00:00 2001 +From: Yuhang Wei +Date: Mon, 12 Aug 2024 19:42:24 +0800 +Subject: [PATCH 3/9] docs: update readme and format yaml + +add kbimg command line explanation and the way to generate root passwd for login KubeOS +format yamls + +Signed-off-by: Yuhang Wei +--- + .../admin-container/admin-container.yaml | 126 +++++++++--------- + docs/example/config/manager/manager.yaml | 58 ++++---- + .../config/samples/upgrade_v1alpha1_os.yaml | 68 +++++----- + docs/quick-start.md | 122 ++++++++++------- + ...66\344\275\234\346\214\207\345\257\274.md" | 71 +++++----- + 5 files changed, 238 insertions(+), 207 deletions(-) + +diff --git a/docs/example/config/admin-container/admin-container.yaml b/docs/example/config/admin-container/admin-container.yaml +index b5ec9e03..518b955f 100644 +--- a/docs/example/config/admin-container/admin-container.yaml ++++ b/docs/example/config/admin-container/admin-container.yaml +@@ -1,63 +1,63 @@ +-apiVersion: v1 +-kind: Secret +-metadata: +- name: root-secret +-data: +- ssh-pub-key: your-ssh-pub-key +---- +-apiVersion: apps/v1 +-kind: Deployment +-metadata: +- name: admin-container-sysmaster +- namespace: default +- labels: +- control-plane: admin-container-sysmaster +-spec: +- selector: +- matchLabels: +- control-plane: admin-container-sysmaster +- replicas: 1 +- template: +- metadata: +- labels: +- control-plane: admin-container-sysmaster +- spec: +- hostPID: true +- containers: +- - name: admin-container-sysmaster +- image: your_imageRepository/admin_imageName:version +- imagePullPolicy: Always +- securityContext: +- privileged: true +- ports: +- - containerPort: 22 +- # sysmaster要求 +- env: +- - name: container +- value: containerd +- volumeMounts: +- # name 必须与下面的卷名匹配 +- - name: secret-volume +- # mountPath必须为/etc/secret-volume +- mountPath: /etc/secret-volume +- readOnly: true +- nodeName: your-worker-node-name +- volumes: +- - name: secret-volume +- secret: +- # secretName必须与上面指定的Secret的name相同 +- secretName: root-secret +---- +-apiVersion: v1 +-kind: Service +-metadata: +- name: admin-container-sysmaster +- namespace: default +-spec: +- type: NodePort +- ports: +- - port: 22 +- targetPort: 22 +- nodePort: your-exposed-port +- selector: +- control-plane: admin-container-sysmaster +\ No newline at end of file ++apiVersion: v1 ++kind: Secret ++metadata: ++ name: root-secret ++data: ++ ssh-pub-key: ++--- ++apiVersion: apps/v1 ++kind: Deployment ++metadata: ++ name: admin-container-sysmaster ++ namespace: default ++ labels: ++ control-plane: admin-container-sysmaster ++spec: ++ selector: ++ matchLabels: ++ control-plane: admin-container-sysmaster ++ replicas: 1 ++ template: ++ metadata: ++ labels: ++ control-plane: admin-container-sysmaster ++ spec: ++ hostPID: true ++ containers: ++ - name: admin-container-sysmaster ++ image: ++ imagePullPolicy: Always ++ securityContext: ++ privileged: true ++ ports: ++ - containerPort: 22 ++ # sysmaster要求 ++ env: ++ - name: container ++ value: containerd ++ volumeMounts: ++ # name 必须与下面的卷名匹配 ++ - name: secret-volume ++ # mountPath必须为/etc/secret-volume ++ mountPath: /etc/secret-volume ++ readOnly: true ++ nodeName: ++ volumes: ++ - name: secret-volume ++ secret: ++ # secretName必须与上面指定的Secret的name相同 ++ secretName: root-secret ++--- ++apiVersion: v1 ++kind: Service ++metadata: ++ name: admin-container-sysmaster ++ namespace: default ++spec: ++ type: NodePort ++ ports: ++ - port: 22 ++ targetPort: 22 ++ nodePort: ++ selector: ++ control-plane: admin-container-sysmaster +diff --git a/docs/example/config/manager/manager.yaml b/docs/example/config/manager/manager.yaml +index 93d15220..f5cf84f9 100644 +--- a/docs/example/config/manager/manager.yaml ++++ b/docs/example/config/manager/manager.yaml +@@ -20,19 +20,19 @@ spec: + control-plane: upgrade-proxy + spec: + containers: +- - name: proxy +- command: +- - /proxy +- image: edit.proxy.image.addr +- volumeMounts: +- - name: upgrade-agent +- mountPath: /var/run/os-agent +- env: +- - name: NODE_NAME +- valueFrom: +- fieldRef: +- apiVersion: v1 +- fieldPath: spec.nodeName ++ - name: proxy ++ command: ++ - /proxy ++ image: ++ volumeMounts: ++ - name: upgrade-agent ++ mountPath: /var/run/os-agent ++ env: ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ apiVersion: v1 ++ fieldPath: spec.nodeName + volumes: + - name: upgrade-agent + hostPath: +@@ -56,24 +56,26 @@ spec: + control-plane: upgrade-operator + spec: + containers: +- - command: +- - /operator +- image: edit.operator.image.addr +- name: operator +- securityContext: +- allowPrivilegeEscalation: false +- runAsUser: 6552 +- runAsGroup: 6552 +- resources: +- limits: +- cpu: 100m +- memory: 30Mi +- requests: +- cpu: 100m +- memory: 20Mi ++ - command: ++ - /operator ++ image: ++ name: operator ++ securityContext: ++ allowPrivilegeEscalation: false ++ runAsUser: 6552 ++ runAsGroup: 6552 ++ resources: ++ limits: ++ cpu: 100m ++ memory: 30Mi ++ requests: ++ cpu: 100m ++ memory: 20Mi + terminationGracePeriodSeconds: 10 + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" ++ - key: "node-role.kubernetes.io/control-plane" ++ operator: "Exists" +diff --git a/docs/example/config/samples/upgrade_v1alpha1_os.yaml b/docs/example/config/samples/upgrade_v1alpha1_os.yaml +index b33a30b2..d7528831 100644 +--- a/docs/example/config/samples/upgrade_v1alpha1_os.yaml ++++ b/docs/example/config/samples/upgrade_v1alpha1_os.yaml +@@ -1,38 +1,38 @@ + apiVersion: upgrade.openeuler.org/v1alpha1 + kind: OS + metadata: +- name: os-sample ++ name: os-sample + spec: +- imagetype: docker/containerd/disk +- opstype: upgrade/config/rollback +- osversion: edit.os.version +- maxunavailable: edit.node.upgrade.number +- containerimage: "" +- evictpodforce: true +- imageurl: "" +- checksum: image digests +- flagSafe: false +- mtls: false +- sysconfigs: +- version: edit.sysconfig.version +- configs: +- - model: kernel.systcl +- contents: +- - key: kernel param key1 +- value: kernel param value1 +- - key: kernel param key2 +- value: kernel param value2 +- - model: kernel.systcl.persist +- configpath: persist file path +- contents: +- - key: kernel param key3 +- value: kernel param value3 +- operation: delete +- upgradeconfigs: +- version: edit.upgradeconfig.version +- configs: +- - model: kernel.systcl +- contents: +- - key: kernel param key4 +- value: kernel param value4 +- operation: delete +\ No newline at end of file ++ imagetype: docker/containerd/disk ++ opstype: upgrade/config/rollback ++ osversion: edit.os.version ++ maxunavailable: edit.node.upgrade.number ++ containerimage: "" ++ evictpodforce: true ++ imageurl: "" ++ checksum: image digests ++ flagSafe: false ++ mtls: false ++ sysconfigs: ++ version: edit.sysconfig.version ++ configs: ++ - model: kernel.sysctl ++ contents: ++ - key: kernel param key1 ++ value: kernel param value1 ++ - key: kernel param key2 ++ value: kernel param value2 ++ - model: kernel.sysctl.persist ++ configpath: persist file path ++ contents: ++ - key: kernel param key3 ++ value: kernel param value3 ++ operation: delete ++ upgradeconfigs: ++ version: edit.upgradeconfig.version ++ configs: ++ - model: kernel.sysctl ++ contents: ++ - key: kernel param key4 ++ value: kernel param value4 ++ operation: delete +diff --git a/docs/quick-start.md b/docs/quick-start.md +index 9656fb99..372c904b 100644 +--- a/docs/quick-start.md ++++ b/docs/quick-start.md +@@ -10,8 +10,8 @@ + * golang(大于等于1.15版本) + * make + * git +- * rust(大于等于1.57版本) +- * cargo(大于等于1.57版本) ++ * rust(大于等于1.64版本) ++ * cargo(大于等于1.64版本) + * openssl-devel + + ``` shell +@@ -65,11 +65,18 @@ + + * Dockerfile参考如下, Dockerfile也可以使用多阶段构建: + ++ `proxy`容器镜像Dockerfile ++ + ``` dockerfile +- FROM your_baseimage ++ FROM openeuler/openeuler:24.03-lts + COPY ./bin/proxy /proxy + ENTRYPOINT ["/proxy"] +- FROM your_baseimage ++ ``` ++ ++ `operator`容器镜像Dockerfile ++ ++ ``` dockerfile ++ FROM openeuler/openeuler:24.03-lts + COPY --chown=6552:6552 ./bin/operator /operator + ENTRYPOINT ["/operator"] + ``` +@@ -112,14 +119,34 @@ + bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' + ``` + +- * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 +- * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： +- * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 +- * qcow2 格式的系统镜像 system.qcow2。 +- * 可用于升级的根文件系统分区镜像 update.img 。 +- * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 +- * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) +- * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 ++ 参数说明如下: ++ ++ ```bash ++ Usage : kbimg create vm-image -p iso-path -v os-version -b os-agent-dir -e os-password ++ or ++ kbimg create vm-image -d repository/name:tag ++ ++ options: ++ -p repo path ++ -v KubeOS version ++ -b path of os-agent binary ++ -e os encrypted password ++ -d docker image like repository/name:tag ++ -l boot to legacy BIOS mode, if not specify, then UEFI mode ++ -h,--help show help information ++ ``` ++ ++ * 其中 xxx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 ++ * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： ++ * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 ++ * qcow2 格式的系统镜像 system.qcow2。 ++ * 可用于升级的根文件系统分区镜像 update.img 。 ++ * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景。若x86 架构的虚拟机需要使用 legacy 启动模式，需制作镜像时指定-l参数 ++ * 默认root密码为openEuler12#$ ++ * 您可通过`openssl passwd -6 -salt $(head -c18 /dev/urandom | openssl base64)`命令生成root密码并通过`-e`参数配置密码 ++ * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) ++ * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 ++ * 详细参数说明请见[《容器OS镜像制作指导》](../docs/user_guide/%E5%AE%B9%E5%99%A8OS%E9%95%9C%E5%83%8F%E5%88%B6%E4%BD%9C%E6%8C%87%E5%AF%BC.md) + + * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 + +@@ -173,7 +200,7 @@ + + ## 使用指导 + +-#### 注意事项 ++### 注意事项 + + * 公共注意事项 + * 仅支持虚拟机x86和arm64 UEFI场景。 +@@ -209,7 +236,7 @@ + + | 参数 |参数类型 | 参数说明 | 使用说明 | 是否必选 | + | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- | +- | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | ++ | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | + | opstype | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是 | + | osversion | string | 升级/回退的目标版本 | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是 | + | maxunavailable | int | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是 | +@@ -357,13 +384,13 @@ + kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' + ``` + +-* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 ++* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 + + #### 配置（Settings）指导 + + * Settings参数说明: + +- 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： ++ 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： + + ```yaml + apiVersion: upgrade.openeuler.org/v1alpha1 +@@ -424,9 +451,7 @@ + | value | string | 参数值 | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 | + | operation | string | 对参数进行的操作 | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否 | + +- +- +- * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 ++ * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 + + * 使用说明 + +@@ -463,7 +488,7 @@ + 2. upgrade模式重新升级至上一版本 + * 手动回退指导 + +- * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 ++ * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 + * 工具回退指导 + * 回退至任意版本 + * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 +@@ -554,19 +579,14 @@ KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器， + + ```Dockerfile + FROM openeuler-22.03-lts-sp1 +- + RUN yum -y install openssh-clients util-linux +- + ADD ./your-sysmaster.rpm /home + RUN rpm -ivh /home/your-sysmaster.rpm +- + COPY ./hostshell /usr/bin/ + COPY ./set-ssh-pub-key.sh /usr/local/bin + COPY ./set-ssh-pub-key.service /usr/lib/sysmaster +- + EXPOSE 22 + RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target +- + CMD ["/usr/lib/sysmaster/init"] + ``` + +@@ -594,8 +614,7 @@ kind: Secret + metadata: + name: root-secret + data: +- # base64 encode your pub key in one line +- ssh-pub-key: your-ssh-pub-key ++ ssh-pub-key: + --- + apiVersion: apps/v1 + kind: Deployment +@@ -603,7 +622,7 @@ metadata: + name: admin-container-sysmaster + namespace: default + labels: +- control-plane: admin-container-sysmaster ++ control-plane: admin-container-sysmaster + spec: + selector: + matchLabels: +@@ -616,24 +635,24 @@ spec: + spec: + hostPID: true + containers: +- - name: admin-container-sysmaster +- image: your_imageRepository/admin_imageName:version +- imagePullPolicy: Always +- securityContext: +- privileged: true +- ports: +- - containerPort: 22 +- # sysmaster要求 +- env: +- - name: container +- value: containerd +- volumeMounts: +- # name 必须与下面的卷名匹配 +- - name: secret-volume +- # mountPath必须为/etc/secret-volume +- mountPath: /etc/secret-volume +- readOnly: true +- nodeName: your-worker-node-name ++ - name: admin-container-sysmaster ++ image: ++ imagePullPolicy: Always ++ securityContext: ++ privileged: true ++ ports: ++ - containerPort: 22 ++ # sysmaster要求 ++ env: ++ - name: container ++ value: containerd ++ volumeMounts: ++ # name 必须与下面的卷名匹配 ++ - name: secret-volume ++ # mountPath必须为/etc/secret-volume ++ mountPath: /etc/secret-volume ++ readOnly: true ++ nodeName: + volumes: + - name: secret-volume + secret: +@@ -650,9 +669,9 @@ spec: + ports: + - port: 22 + targetPort: 22 +- nodePort: your-exposed-port ++ nodePort: + selector: +- control-plane: admin-container-sysmaster ++ control-plane: admin-container-sysmaster + ``` + + ### admin容器使用 +@@ -680,6 +699,7 @@ hostshell + #### kernel Settings + + * kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ++ + ```yaml + configs: + - model: kernel.sysctl +@@ -690,7 +710,9 @@ hostshell + value: 0 + operation: delete + ``` ++ + * kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ++ + ```yaml + configs: + - model: kernel.sysctl.persist +@@ -713,8 +735,8 @@ hostshell + + * KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： + +- - grub.cmdline.current：对当前分区的启动项参数进行配置。 +- - grub.cmdline.next：对下一分区的启动项参数进行配置。 ++ * grub.cmdline.current：对当前分区的启动项参数进行配置。 ++ * grub.cmdline.next：对下一分区的启动项参数进行配置。 + + * 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 + +diff --git "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" +index f2823c85..155bc96f 100644 +--- "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" ++++ "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" +@@ -1,16 +1,16 @@ +-# 容器OS镜像制作指导# ++# 容器OS镜像制作指导 + +-## 简介 ## ++## 简介 + + kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制作KubeOS 容器，虚拟机和物理机镜像 + +-## 命令介绍 ## ++## 命令介绍 + +-### 命令格式 ### ++### 命令格式 + + **bash kbimg.sh** \[ --help | -h \] create \[ COMMANDS \] \[ OPTIONS \] + +-### 参数说明 ### ++### 参数说明 + + * COMMANDS + +@@ -20,8 +20,6 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制 + | vm-image | 生成用于部署和升级的虚拟机镜像 | + | pxe-image | 生成物理机安装所需的镜像及文件 | + +- +- + * OPTIONS + + | 参数 | 描述 | +@@ -34,31 +32,33 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制 + | -l | 如果指定参数，则镜像为legacy引导，不指定默认是UEFI引导 | + | -h --help | 查看帮助信息 | + ++## 使用说明 + +- +-## 使用说明 ## +- +-#### 注意事项 ### ++### 注意事项 + + * kbimg.sh 执行需要 root 权限 + * 当前仅支持 x86和 AArch64 架构使用 + * 容器 OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 + +-### KubeOS OCI 镜像制作 ### ++### KubeOS OCI 镜像制作 + +-#### 注意事项 #### ++#### 注意事项 + + * 制作的 OCI 镜像仅用于后续的虚拟机/物理机镜像制作或升级使用，不支持启动容器 + * 使用默认 rpmlist 进行容器OS镜像制作时所需磁盘空间至少为6G，如自已定义 rpmlist 可能会超过6G + +-#### 使用示例 #### ++#### 使用示例 ++ + * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 ++ + ```shell + cd /opt/kubeOS/scripts + touch resolv.conf + vim resolv.conf + ``` ++ + * 制作KubeOS容器镜像 ++ + ``` shell + cd /opt/kubeOS/scripts + bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' -d your_imageRepository/imageName:version +@@ -70,25 +70,28 @@ bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1 + docker images + ``` + +-### KubeOS 虚拟机镜像制作 ### ++### KubeOS 虚拟机镜像制作 + +-#### 注意事项 #### ++#### 注意事项 + + * 如使用 docker 镜像制作请先拉取相应镜像或者先制作docker镜像，并保证 docker 镜像的安全性 + * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机 + * 容器 OS 目前不支持 x86 架构的虚拟机使用 legacy 启动模式启动 + * 使用默认rpmlist进行容器OS镜像制作时所需磁盘空间至少为25G，如自已定义rpmlist可能会超过25G + +-#### 使用示例 #### ++#### 使用示例 + + * 使用repo源制作 +- * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 ++ * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 ++ + ```shell + cd /opt/kubeOS/scripts + touch resolv.conf + vim resolv.conf + ``` +- * KubeOS虚拟机镜像制作 ++ ++ * KubeOS虚拟机镜像制作 ++ + ``` shell + cd /opt/kubeOS/scripts + bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' +@@ -100,15 +103,15 @@ docker images + cd /opt/kubeOS/scripts + bash kbimg.sh create vm-image -d your_imageRepository/imageName:version + ``` +-* 结果说明 +- 容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成： +- * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 +- * update.img: 用于升级的根文件系统分区镜像 + ++* 结果说明 ++ 容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成： ++ * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 ++ * update.img: 用于升级的根文件系统分区镜像 + +-### KubeOS 物理机安装所需镜像及文件制作 ### ++### KubeOS 物理机安装所需镜像及文件制作 + +-#### 注意事项 #### ++#### 注意事项 + + * 如使用 docker 镜像制作请先拉取相应镜像或者先制作 docker 镜像，并保证 docker 镜像的安全性 + * 制作出来的容器 OS 物理安装所需的镜像目前只能用于 CPU 架构为 x86 和 AArch64 的物理机安装 +@@ -116,7 +119,8 @@ docker images + * 不支持多个磁盘都安装KubeOS，可能会造成启动失败或挂载紊乱 + * 容器OS 目前不支持 x86 架构的物理机使用 legacy 启动模式启动 + * 使用默认rpmlist进行镜像制作时所需磁盘空间至少为5G，如自已定义 rpmlist 可能会超过5G +-#### 使用示例 #### ++ ++#### 使用示例 + + * 首先需要修改```00bootup/Global.cfg```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下 + +@@ -138,25 +142,28 @@ docker images + ``` + + * 使用 repo 源制作 +- * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件 ++ * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件 ++ + ```shell + cd /opt/kubeOS/scripts + touch resolv.conf + vim resolv.conf + ``` +- * KubeOS物理机安装所需镜像制作 +- ``` ++ ++ * KubeOS物理机安装所需镜像制作 ++ ++ ```shell + cd /opt/kubeOS/scripts + bash kbimg.sh create pxe-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' + ``` + + * 使用 docker 镜像制作 ++ + ``` shell + cd /opt/kubeOS/scripts + bash kbimg.sh create pxe-image -d your_imageRepository/imageName:version + ``` + + * 结果说明 +- +- * initramfs.img: 用于pxe启动用的 initramfs 镜像 +- * kubeos.tar: pxe安装所用的 OS ++ * initramfs.img: 用于pxe启动用的 initramfs 镜像 ++ * kubeos.tar: pxe安装所用的 OS +-- +2.33.0.windows.2 + diff --git a/0004-KubeOS-add-nodeselector-to-OS.patch b/0004-KubeOS-add-nodeselector-to-OS.patch new file mode 100644 index 0000000..2adeeb3 --- /dev/null +++ b/0004-KubeOS-add-nodeselector-to-OS.patch @@ -0,0 +1,1230 @@ +From 4d8d07df49e44202d252344b25e8b1d366746ecc Mon Sep 17 00:00:00 2001 +From: liyuanr +Date: Mon, 25 Sep 2023 10:52:29 +0800 +Subject: [PATCH 4/9] KubeOS:add nodeselector to OS + +The nodeselector field is added to the OS to filter the +nodes to be upgraded or configured. +During nodeselector configuration, only nodes with this +label can be upgraded or configured. + +Signed-off-by: liyuanr +--- + api/v1alpha1/os_types.go | 2 + + cmd/operator/controllers/operation.go | 181 ++++++ + cmd/operator/controllers/os_controller.go | 228 +++----- + .../controllers/os_controller_test.go | 550 ++++++++++++++---- + cmd/proxy/controllers/os_controller.go | 7 + + .../config/crd/upgrade.openeuler.org_os.yaml | 2 + + docs/quick-start.md | 37 +- + pkg/values/values.go | 6 +- + 8 files changed, 736 insertions(+), 277 deletions(-) + create mode 100644 cmd/operator/controllers/operation.go + +diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go +index f9474b72..d3d636de 100644 +--- a/api/v1alpha1/os_types.go ++++ b/api/v1alpha1/os_types.go +@@ -38,6 +38,8 @@ type OSSpec struct { + SysConfigs SysConfigs `json:"sysconfigs"` + // +kubebuilder:validation:Optional + UpgradeConfigs SysConfigs `json:"upgradeconfigs"` ++ // +kubebuilder:validation:Optional ++ NodeSelector string `json:"nodeselector"` + } + + // +kubebuilder:subresource:status +diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go +new file mode 100644 +index 00000000..4b441d18 +--- /dev/null ++++ b/cmd/operator/controllers/operation.go +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * KubeOS is licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ */ ++ ++// Package controllers contains the Reconcile of operator ++package controllers ++ ++import ( ++ "context" ++ ++ corev1 "k8s.io/api/core/v1" ++ "k8s.io/apimachinery/pkg/labels" ++ "k8s.io/apimachinery/pkg/selection" ++ "k8s.io/apimachinery/pkg/types" ++ upgradev1 "openeuler.org/KubeOS/api/v1alpha1" ++ "openeuler.org/KubeOS/pkg/common" ++ "openeuler.org/KubeOS/pkg/values" ++ "sigs.k8s.io/controller-runtime/pkg/client" ++) ++ ++type operation interface { ++ newExistRequirement() (labels.Requirement, error) ++ newNotExistRequirement() (labels.Requirement, error) ++ updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ nodes []corev1.Node, limit int) (int, error) ++ updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ node *corev1.Node, osInstance *upgradev1.OSInstance) error ++} ++ ++type upgradeOps struct{} ++ ++func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelUpgrading) ++ return labels.Requirement{}, err ++ } ++ return *requirement, nil ++} ++ ++func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelUpgrading) ++ return labels.Requirement{}, err ++ } ++ return *requirement, nil ++} ++ ++func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ nodes []corev1.Node, limit int) (int, error) { ++ var count int ++ for _, node := range nodes { ++ if count >= limit { ++ break ++ } ++ osVersionNode := node.Status.NodeInfo.OSImage ++ if os.Spec.OSVersion != osVersionNode { ++ var osInstance upgradev1.OSInstance ++ if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { ++ if err = client.IgnoreNotFound(err); err != nil { ++ log.Error(err, "failed to get osInstance "+node.Name, "skip this node") ++ return count, err ++ } ++ continue ++ } ++ if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { ++ log.Error(err, "failed to update node and osinstance ,skip this node ") ++ continue ++ } ++ count++ ++ } ++ } ++ return count, nil ++} ++ ++func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ node *corev1.Node, osInstance *upgradev1.OSInstance) error { ++ if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { ++ if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { ++ return err ++ } ++ } ++ if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { ++ if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { ++ return err ++ } ++ // exchange "grub.cmdline.current" and "grub.cmdline.next" ++ for i, config := range osInstance.Spec.SysConfigs.Configs { ++ if config.Model == "grub.cmdline.current" { ++ osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" ++ } ++ if config.Model == "grub.cmdline.next" { ++ osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" ++ } ++ } ++ } ++ osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() ++ if err := r.Update(ctx, osInstance); err != nil { ++ log.Error(err, "unable to update", "osInstance", osInstance.Name) ++ return err ++ } ++ node.Labels[values.LabelUpgrading] = "" ++ if err := r.Update(ctx, node); err != nil { ++ log.Error(err, "unable to label", "node", node.Name) ++ return err ++ } ++ return nil ++} ++ ++type configOps struct{} ++ ++func (c configOps) newExistRequirement() (labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelConfiguring) ++ return labels.Requirement{}, err ++ } ++ return *requirement, nil ++} ++ ++func (c configOps) newNotExistRequirement() (labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelConfiguring) ++ return labels.Requirement{}, err ++ } ++ return *requirement, nil ++} ++ ++func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ nodes []corev1.Node, limit int) (int, error) { ++ var count int ++ for _, node := range nodes { ++ if count >= limit { ++ break ++ } ++ var osInstance upgradev1.OSInstance ++ if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { ++ if err = client.IgnoreNotFound(err); err != nil { ++ log.Error(err, "failed to get osInstance "+node.Name) ++ return count, err ++ } ++ continue ++ } ++ if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { ++ if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { ++ log.Error(err, "failed to update node and osinstance ,skip this node ") ++ continue ++ } ++ count++ ++ } ++ } ++ return count, nil ++} ++ ++func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ node *corev1.Node, osInstance *upgradev1.OSInstance) error { ++ if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { ++ return err ++ } ++ osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() ++ if err := r.Update(ctx, osInstance); err != nil { ++ log.Error(err, "unable to update", "osInstance", osInstance.Name) ++ return err ++ } ++ node.Labels[values.LabelConfiguring] = "" ++ if err := r.Update(ctx, node); err != nil { ++ log.Error(err, "unable to label", "node", node.Name) ++ return err ++ } ++ return nil ++} +diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go +index e04d59b1..bd7a70dd 100644 +--- a/cmd/operator/controllers/os_controller.go ++++ b/cmd/operator/controllers/os_controller.go +@@ -17,6 +17,7 @@ import ( + "context" + "encoding/json" + "fmt" ++ "reflect" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" +@@ -65,29 +66,24 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) + } + + ops := os.Spec.OpsType ++ var opsInsatnce operation + switch ops { + case "upgrade", "rollback": +- limit, err := checkUpgrading(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) // adjust maxUnavailable if need +- if err != nil { +- return values.RequeueNow, err +- } +- if needRequeue, err := assignUpgrade(ctx, r, os, limit, req.Namespace); err != nil { +- return values.RequeueNow, err +- } else if needRequeue { +- return values.Requeue, nil +- } ++ opsInsatnce = upgradeOps{} + case "config": +- limit, err := checkConfig(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) +- if err != nil { +- return values.RequeueNow, err +- } +- if needRequeue, err := assignConfig(ctx, r, os.Spec.SysConfigs, os.Spec.SysConfigs.Version, limit); err != nil { +- return values.RequeueNow, err +- } else if needRequeue { +- return values.Requeue, nil +- } ++ opsInsatnce = configOps{} + default: + log.Error(nil, "operation "+ops+" cannot be recognized") ++ return values.Requeue, nil ++ } ++ limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need ++ if err != nil { ++ return values.RequeueNow, err ++ } ++ if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { ++ return values.RequeueNow, err ++ } else if needRequeue { ++ return values.Requeue, nil + } + return values.Requeue, nil + } +@@ -129,30 +125,44 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim + } + } + +-func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (os upgradev1.OS, +- nodeNum int, err error) { +- if err = r.Get(ctx, name, &os); err != nil { ++func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, ++ int, error) { ++ var os upgradev1.OS ++ if err := r.Get(ctx, name, &os); err != nil { + log.Error(err, "unable to fetch OS") +- return ++ return upgradev1.OS{}, 0, err + } + + requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelMaster) +- return ++ return upgradev1.OS{}, 0, err + } +- nodesItems, err := getNodes(ctx, r, 0, *requirement) ++ var requirements []labels.Requirement ++ requirements = append(requirements, *requirement) ++ if os.Spec.NodeSelector != "" { ++ reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return upgradev1.OS{}, 0, err ++ } ++ requirements = append(requirements, *requirement, *reqSelector) ++ } ++ nodesItems, err := getNodes(ctx, r, 0, requirements...) + if err != nil { + log.Error(err, "get slave nodes fail") +- return ++ return upgradev1.OS{}, 0, err + } +- nodeNum = len(nodesItems) +- return ++ nodeNum := len(nodesItems) ++ return os, nodeNum, nil + } + +-func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, +- nameSpace string) (bool, error) { +- requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) ++func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, ++ ops operation) (bool, error) { ++ fmt.Println("start assignOperation") ++ fmt.Println("ops is ", reflect.TypeOf(ops)) ++ requirement, err := ops.newNotExistRequirement() ++ fmt.Println("requirement is ", requirement.String()) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return false, err +@@ -162,14 +172,25 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. + log.Error(err, "unable to create requirement "+values.LabelMaster) + return false, err + } ++ var requirements []labels.Requirement ++ requirements = append(requirements, requirement, *reqMaster) ++ if os.Spec.NodeSelector != "" { ++ reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return false, err ++ } ++ fmt.Println("requirement is ", reqSelector.String()) ++ requirements = append(requirements, *reqSelector) ++ } + +- nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all nodes updated ++ nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated + if err != nil { + return false, err + } +- ++ fmt.Println("nodes has not upgrade/config and has selector ", len(nodes)) + // Upgrade OS for selected nodes +- count, err := upgradeNodes(ctx, r, &os, nodes, limit) ++ count, err := ops.updateNodes(ctx, r, &os, nodes, limit) + if err != nil { + return false, err + } +@@ -177,90 +198,6 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. + return count >= limit, nil + } + +-func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +- nodes []corev1.Node, limit int) (int, error) { +- var count int +- for _, node := range nodes { +- if count >= limit { +- break +- } +- osVersionNode := node.Status.NodeInfo.OSImage +- if os.Spec.OSVersion != osVersionNode { +- var osInstance upgradev1.OSInstance +- if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { +- if err = client.IgnoreNotFound(err); err != nil { +- log.Error(err, "failed to get osInstance "+node.Name) +- return count, err +- } +- continue +- } +- if err := updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { +- continue +- } +- count++ +- } +- } +- return count, nil +-} +- +-func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +- node *corev1.Node, osInstance *upgradev1.OSInstance) error { +- if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { +- if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { +- return err +- } +- } +- if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { +- if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { +- return err +- } +- // exchange "grub.cmdline.current" and "grub.cmdline.next" +- for i, config := range osInstance.Spec.SysConfigs.Configs { +- if config.Model == "grub.cmdline.current" { +- osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" +- } +- if config.Model == "grub.cmdline.next" { +- osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" +- } +- } +- } +- osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() +- if err := r.Update(ctx, osInstance); err != nil { +- log.Error(err, "unable to update", "osInstance", osInstance.Name) +- return err +- } +- node.Labels[values.LabelUpgrading] = "" +- if err := r.Update(ctx, node); err != nil { +- log.Error(err, "unable to label", "node", node.Name) +- return err +- } +- return nil +-} +- +-func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, +- configVersion string, limit int) (bool, error) { +- osInstances, err := getIdleOSInstances(ctx, r, limit+1) // one more to see if all node updated +- if err != nil { +- return false, err +- } +- var count = 0 +- for _, osInstance := range osInstances { +- if count >= limit { +- break +- } +- configVersionNode := osInstance.Spec.SysConfigs.Version +- if configVersion != configVersionNode { +- count++ +- osInstance.Spec.SysConfigs = sysConfigs +- osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() +- if err = r.Update(ctx, &osInstance); err != nil { +- log.Error(err, "unable update osInstance ", "osInstanceName ", osInstance.Name) +- } +- } +- } +- return count >= limit, nil +-} +- + func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, + reqs ...labels.Requirement) ([]corev1.Node, error) { + var nodeList corev1.NodeList +@@ -272,48 +209,37 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, + return nodeList.Items, nil + } + +-func getIdleOSInstances(ctx context.Context, r common.ReadStatusWriter, limit int) ([]upgradev1.OSInstance, error) { +- var osInstanceList upgradev1.OSInstanceList +- opt := []client.ListOption{ +- client.MatchingFields{values.OsiStatusName: values.NodeStatusIdle.String()}, +- &client.ListOptions{Limit: int64(limit)}, +- } +- if err := r.List(ctx, &osInstanceList, opt...); err != nil { +- log.Error(err, "unable to list nodes with requirements") +- return nil, err +- } +- return osInstanceList.Items, nil +-} +- +-func getConfigOSInstances(ctx context.Context, r common.ReadStatusWriter) ([]upgradev1.OSInstance, error) { +- var osInstanceList upgradev1.OSInstanceList +- if err := r.List(ctx, &osInstanceList, +- client.MatchingFields{values.OsiStatusName: values.NodeStatusConfig.String()}); err != nil { +- log.Error(err, "unable to list nodes with requirements") +- return nil, err +- } +- return osInstanceList.Items, nil +-} +- +-func checkUpgrading(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { +- requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) ++func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, ++ ops operation, maxUnavailable int, nodeSelector string) (int, error) { ++ fmt.Println("start calNodeLimit") ++ fmt.Println("ops is ", reflect.TypeOf(ops)) ++ requirement, err := ops.newExistRequirement() + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return 0, err + } +- nodes, err := getNodes(ctx, r, 0, *requirement) +- if err != nil { +- return 0, err ++ fmt.Println("requirement is ", requirement.String()) ++ var requirements []labels.Requirement ++ requirements = append(requirements, requirement) ++ if nodeSelector != "" { ++ reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return 0, err ++ } ++ fmt.Println("requirement is ", reqSelector.String()) ++ requirements = append(requirements, *reqSelector) + } +- return maxUnavailable - len(nodes), nil +-} +- +-func checkConfig(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { +- osInstances, err := getConfigOSInstances(ctx, r) ++ nodes, err := getNodes(ctx, r, 0, requirements...) + if err != nil { + return 0, err ++ + } +- return maxUnavailable - len(osInstances), nil ++ fmt.Println("nodes has upgrade and selector ", len(nodes)) ++ for _, n := range nodes { ++ fmt.Println(" nodes name is ", n.Name) ++ } ++ return maxUnavailable - len(nodes), nil + } + + func min(a, b int) int { +diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go +index 8c5d1981..38bf4e61 100644 +--- a/cmd/operator/controllers/os_controller_test.go ++++ b/cmd/operator/controllers/os_controller_test.go +@@ -455,6 +455,14 @@ var _ = Describe("OsController", func() { + }, timeout, interval).Should(BeTrue()) + Expect(configedOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(configedOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) ++ existingNode1 := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode1) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ _, ok := existingNode1.Labels[values.LabelConfiguring] ++ Expect(ok).Should(Equal(true)) + + configedOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { +@@ -463,6 +471,14 @@ var _ = Describe("OsController", func() { + }, timeout, interval).Should(BeTrue()) + Expect(configedOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(configedOSIns2.Spec.SysConfigs.Version).Should(Equal("v2")) ++ existingNode2 := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ _, ok = existingNode2.Labels[values.LabelConfiguring] ++ Expect(ok).Should(Equal(true)) + }) + }) + +@@ -749,6 +765,419 @@ var _ = Describe("OsController", func() { + Expect(createdOS.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + }) + }) ++ ++ Context("When we want to upgrade node with nodes having NodeSelector label", func() { ++ It("Should only update node with NodeSelector label", func() { ++ ctx := context.Background() ++ // create Node1 ++ node1Name = "test-node-" + uuid.New().String() ++ node1 := &v1.Node{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node1Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ "beta.kubernetes.io/os": "linux", ++ "upgrade.openeuler.org/node-selector": "openeuler", ++ }, ++ }, ++ TypeMeta: metav1.TypeMeta{ ++ APIVersion: "v1", ++ Kind: "Node", ++ }, ++ Status: v1.NodeStatus{ ++ NodeInfo: v1.NodeSystemInfo{ ++ OSImage: "KubeOS v1", ++ }, ++ }, ++ } ++ err := k8sClient.Create(ctx, node1) ++ Expect(err).ToNot(HaveOccurred()) ++ existingNode := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ ++ // create OSInstance1 ++ OSIns := &upgradev1.OSInstance{ ++ TypeMeta: metav1.TypeMeta{ ++ Kind: "OSInstance", ++ APIVersion: "upgrade.openeuler.org/v1alpha1", ++ }, ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node1Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ values.LabelOSinstance: node1Name, ++ }, ++ }, ++ Spec: upgradev1.OSInstanceSpec{ ++ SysConfigs: upgradev1.SysConfigs{ ++ Version: "v1", ++ Configs: []upgradev1.SysConfig{}, ++ }, ++ UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, ++ NodeStatus: values.NodeStatusIdle.String(), ++ }, ++ } ++ Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) ++ ++ osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} ++ createdOSIns := &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) ++ ++ // create Node2 ++ node2Name := "test-node-" + uuid.New().String() ++ node2 := &v1.Node{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node2Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ "beta.kubernetes.io/os": "linux", ++ }, ++ }, ++ TypeMeta: metav1.TypeMeta{ ++ APIVersion: "v1", ++ Kind: "Node", ++ }, ++ Status: v1.NodeStatus{ ++ NodeInfo: v1.NodeSystemInfo{ ++ OSImage: "KubeOS v1", ++ }, ++ }, ++ } ++ err = k8sClient.Create(ctx, node2) ++ Expect(err).ToNot(HaveOccurred()) ++ existingNode = &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ ++ // create OSInstance2 ++ OSIns = &upgradev1.OSInstance{ ++ TypeMeta: metav1.TypeMeta{ ++ Kind: "OSInstance", ++ APIVersion: "upgrade.openeuler.org/v1alpha1", ++ }, ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node2Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ values.LabelOSinstance: node2Name, ++ }, ++ }, ++ Spec: upgradev1.OSInstanceSpec{ ++ SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, ++ UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, ++ NodeStatus: values.NodeStatusIdle.String(), ++ }, ++ } ++ Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) ++ ++ // Check that the corresponding OSIns CR has been created ++ osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} ++ createdOSIns = &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) ++ ++ OS := &upgradev1.OS{ ++ TypeMeta: metav1.TypeMeta{ ++ APIVersion: "upgrade.openeuler.org/v1alpha1", ++ Kind: "OS", ++ }, ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: OSName, ++ Namespace: testNamespace, ++ }, ++ Spec: upgradev1.OSSpec{ ++ OpsType: "upgrade", ++ MaxUnavailable: 3, ++ OSVersion: "KubeOS v2", ++ FlagSafe: true, ++ MTLS: false, ++ EvictPodForce: true, ++ NodeSelector: "openeuler", ++ SysConfigs: upgradev1.SysConfigs{ ++ Version: "v2", ++ Configs: []upgradev1.SysConfig{ ++ { ++ Model: "kernel.sysctl", ++ Contents: []upgradev1.Content{ ++ {Key: "key1", Value: "a"}, ++ {Key: "key2", Value: "b"}, ++ }, ++ }, ++ }, ++ }, ++ UpgradeConfigs: upgradev1.SysConfigs{ ++ Version: "v2", ++ Configs: []upgradev1.SysConfig{ ++ {Model: "kernel.sysctl.persist", ++ Contents: []upgradev1.Content{ ++ {Key: "key1", Value: "a"}, ++ {Key: "key2", Value: "b"}, ++ }, ++ }}, ++ }, ++ }, ++ } ++ Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) ++ ++ osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} ++ createdOS := &upgradev1.OS{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osCRLookupKey, createdOS) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) ++ ++ time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished ++ existingNode1 := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ _, ok := existingNode1.Labels[values.LabelUpgrading] ++ Expect(ok).Should(Equal(true)) ++ ++ upgradeOSIns1 := &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) ++ Expect(upgradeOSIns1.Spec.UpgradeConfigs.Version).Should(Equal("v2")) ++ Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) ++ ++ existingNode2 := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ _, ok = existingNode2.Labels[values.LabelUpgrading] ++ Expect(ok).Should(Equal(false)) ++ ++ upgradeOSIns2 := &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) ++ Expect(upgradeOSIns2.Spec.UpgradeConfigs.Version).Should(Equal("v1")) ++ Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) ++ }) ++ }) ++ ++ Context("When we want to config node with nodes having NodeSelector label", func() { ++ It("Should only config node with NodeSelector label", func() { ++ ctx := context.Background() ++ // create Node1 ++ node1Name = "test-node-" + uuid.New().String() ++ node1 := &v1.Node{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node1Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ "beta.kubernetes.io/os": "linux", ++ "upgrade.openeuler.org/node-selector": "openeuler", ++ }, ++ }, ++ TypeMeta: metav1.TypeMeta{ ++ APIVersion: "v1", ++ Kind: "Node", ++ }, ++ Status: v1.NodeStatus{ ++ NodeInfo: v1.NodeSystemInfo{ ++ OSImage: "KubeOS v1", ++ }, ++ }, ++ } ++ err := k8sClient.Create(ctx, node1) ++ Expect(err).ToNot(HaveOccurred()) ++ existingNode := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ ++ // create OSInstance1 ++ OSIns := &upgradev1.OSInstance{ ++ TypeMeta: metav1.TypeMeta{ ++ Kind: "OSInstance", ++ APIVersion: "upgrade.openeuler.org/v1alpha1", ++ }, ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node1Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ values.LabelOSinstance: node1Name, ++ }, ++ }, ++ Spec: upgradev1.OSInstanceSpec{ ++ SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, ++ UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, ++ NodeStatus: values.NodeStatusIdle.String(), ++ }, ++ } ++ Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) ++ ++ osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} ++ createdOSIns := &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) ++ ++ // create Node2 ++ node2Name := "test-node-" + uuid.New().String() ++ node2 := &v1.Node{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node2Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ "beta.kubernetes.io/os": "linux", ++ }, ++ }, ++ TypeMeta: metav1.TypeMeta{ ++ APIVersion: "v1", ++ Kind: "Node", ++ }, ++ Status: v1.NodeStatus{ ++ NodeInfo: v1.NodeSystemInfo{ ++ OSImage: "KubeOS v1", ++ }, ++ }, ++ } ++ err = k8sClient.Create(ctx, node2) ++ Expect(err).ToNot(HaveOccurred()) ++ existingNode = &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ ++ // create OSInstance2 ++ OSIns = &upgradev1.OSInstance{ ++ TypeMeta: metav1.TypeMeta{ ++ Kind: "OSInstance", ++ APIVersion: "upgrade.openeuler.org/v1alpha1", ++ }, ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: node2Name, ++ Namespace: testNamespace, ++ Labels: map[string]string{ ++ values.LabelOSinstance: node2Name, ++ }, ++ }, ++ Spec: upgradev1.OSInstanceSpec{ ++ SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, ++ UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, ++ NodeStatus: values.NodeStatusIdle.String(), ++ }, ++ } ++ Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) ++ ++ // Check that the corresponding OSIns CR has been created ++ osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} ++ createdOSIns = &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) ++ ++ OS := &upgradev1.OS{ ++ TypeMeta: metav1.TypeMeta{ ++ APIVersion: "upgrade.openeuler.org/v1alpha1", ++ Kind: "OS", ++ }, ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: OSName, ++ Namespace: testNamespace, ++ }, ++ Spec: upgradev1.OSSpec{ ++ OpsType: "config", ++ MaxUnavailable: 3, ++ OSVersion: "KubeOS v1", ++ FlagSafe: true, ++ MTLS: false, ++ EvictPodForce: true, ++ NodeSelector: "openeuler", ++ SysConfigs: upgradev1.SysConfigs{ ++ Version: "v2", ++ Configs: []upgradev1.SysConfig{ ++ { ++ Model: "kernel.sysctl", ++ Contents: []upgradev1.Content{ ++ {Key: "key1", Value: "a"}, ++ {Key: "key2", Value: "b"}, ++ }, ++ }, ++ }, ++ }, ++ }, ++ } ++ Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) ++ ++ osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} ++ createdOS := &upgradev1.OS{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osCRLookupKey, createdOS) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(createdOS.Spec.SysConfigs.Version).Should(Equal("v2")) ++ ++ time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished ++ existingNode1 := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ _, ok := existingNode1.Labels[values.LabelConfiguring] ++ Expect(ok).Should(Equal(true)) ++ ++ upgradeOSIns1 := &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) ++ Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) ++ ++ existingNode2 := &v1.Node{} ++ Eventually(func() bool { ++ err := k8sClient.Get(context.Background(), ++ types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ _, ok = existingNode2.Labels[values.LabelConfiguring] ++ Expect(ok).Should(Equal(false)) ++ ++ upgradeOSIns2 := &upgradev1.OSInstance{} ++ Eventually(func() bool { ++ err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) ++ return err == nil ++ }, timeout, interval).Should(BeTrue()) ++ Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) ++ Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) ++ }) ++ }) + }) + + func Test_deepCopySpecConfigs(t *testing.T) { +@@ -780,127 +1209,6 @@ func Test_deepCopySpecConfigs(t *testing.T) { + } + } + +-func Test_getConfigOSInstances(t *testing.T) { +- type args struct { +- ctx context.Context +- r common.ReadStatusWriter +- } +- tests := []struct { +- name string +- args args +- want []upgradev1.OSInstance +- wantErr bool +- }{ +- { +- name: "list error", +- args: args{ +- ctx: context.Background(), +- r: &OSReconciler{}, +- }, +- want: nil, +- wantErr: true, +- }, +- } +- patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ +- {Values: gomonkey.Params{fmt.Errorf("list error")}}, +- }) +- defer patchList.Reset() +- for _, tt := range tests { +- t.Run(tt.name, func(t *testing.T) { +- got, err := getConfigOSInstances(tt.args.ctx, tt.args.r) +- if (err != nil) != tt.wantErr { +- t.Errorf("getConfigOSInstances() error = %v, wantErr %v", err, tt.wantErr) +- return +- } +- if !reflect.DeepEqual(got, tt.want) { +- t.Errorf("getConfigOSInstances() = %v, want %v", got, tt.want) +- } +- }) +- } +-} +- +-func Test_checkUpgrading(t *testing.T) { +- type args struct { +- ctx context.Context +- r common.ReadStatusWriter +- maxUnavailable int +- } +- tests := []struct { +- name string +- args args +- want int +- wantErr bool +- }{ +- { +- name: "label error", +- args: args{ +- ctx: context.Background(), +- r: &OSReconciler{}, +- }, +- want: 0, +- wantErr: true, +- }, +- } +- patchNewRequirement := gomonkey.ApplyFuncSeq(labels.NewRequirement, []gomonkey.OutputCell{ +- {Values: gomonkey.Params{nil, fmt.Errorf("label error")}}, +- {Values: gomonkey.Params{nil, nil}}, +- }) +- defer patchNewRequirement.Reset() +- for _, tt := range tests { +- t.Run(tt.name, func(t *testing.T) { +- got, err := checkUpgrading(tt.args.ctx, tt.args.r, tt.args.maxUnavailable) +- if (err != nil) != tt.wantErr { +- t.Errorf("checkUpgrading() error = %v, wantErr %v", err, tt.wantErr) +- return +- } +- if got != tt.want { +- t.Errorf("checkUpgrading() = %v, want %v", got, tt.want) +- } +- }) +- } +-} +- +-func Test_getIdleOSInstances(t *testing.T) { +- type args struct { +- ctx context.Context +- r common.ReadStatusWriter +- limit int +- } +- tests := []struct { +- name string +- args args +- want []upgradev1.OSInstance +- wantErr bool +- }{ +- { +- name: "list error", +- args: args{ +- ctx: context.Background(), +- r: &OSReconciler{}, +- limit: 1, +- }, +- want: nil, +- wantErr: true, +- }, +- } +- patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ +- {Values: gomonkey.Params{fmt.Errorf("list error")}}, +- }) +- defer patchList.Reset() +- for _, tt := range tests { +- t.Run(tt.name, func(t *testing.T) { +- got, err := getIdleOSInstances(tt.args.ctx, tt.args.r, tt.args.limit) +- if (err != nil) != tt.wantErr { +- t.Errorf("getIdleOSInstances() error = %v, wantErr %v", err, tt.wantErr) +- return +- } +- if !reflect.DeepEqual(got, tt.want) { +- t.Errorf("getIdleOSInstances() = %v, want %v", got, tt.want) +- } +- }) +- } +-} +- + func Test_getNodes(t *testing.T) { + type args struct { + ctx context.Context +diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go +index 0630a98c..b543befc 100644 +--- a/cmd/proxy/controllers/os_controller.go ++++ b/cmd/proxy/controllers/os_controller.go +@@ -244,6 +244,13 @@ func (r *OSReconciler) refreshNode(ctx context.Context, node *corev1.Node, osIns + return err + } + } ++ if _, ok := node.Labels[values.LabelConfiguring]; ok { ++ delete(node.Labels, values.LabelConfiguring) ++ if err := r.Update(ctx, node); err != nil { ++ log.Error(err, "unable to delete label", "node", node.Name) ++ return err ++ } ++ } + if node.Spec.Unschedulable { // update done, uncordon the node + drainer := &drain.Helper{ + Ctx: ctx, +diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml +index 2dd822e9..27ff3273 100644 +--- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml ++++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml +@@ -67,6 +67,8 @@ spec: + type: integer + mtls: + type: boolean ++ nodeselector: ++ type: string + opstype: + type: string + osversion: +diff --git a/docs/quick-start.md b/docs/quick-start.md +index 372c904b..af37c224 100644 +--- a/docs/quick-start.md ++++ b/docs/quick-start.md +@@ -251,7 +251,7 @@ + | evictpodforce | bool | 升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | sysconfigs | / | 配置设置 | 1. “opstype=config”时只进行配置。 2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 | + | upgradeconfigs | / | 升级前配置设置 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | +- ++ | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | + #### 升级指导 + + * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如下，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml; +@@ -320,7 +320,7 @@ + mtls: true + ``` + +- * 升级并且进行配置的示例如下， ++ * 升级并且进行配置的示例如下 + * 以节点容器引擎为containerd为例，升级方式对配置无影响，upgradeconfigs在升级前起效，sysconfigs在升级后起效，配置参数说明请见```配置(Settings)指导``` + * 升级并且配置时opstype字段需为upgrade + * upgradeconfig为升级之前执行的配置，sysconfigs为升级机器重启后执行的配置，用户可按需进行配置 +@@ -365,7 +365,37 @@ + - key: kernel param key4 + value: kernel param value4 + ``` ++ * 只升级部分节点示例如下 ++ * 以节点容器引擎为containerd为例，升级方式对节点筛选无影响 ++ * 需要进行升级的节点需包含key为upgrade.openeuler.org/node-selector的label，nodeselector的值为该label的value，即假定nodeselector值为kubeos，则只对包含upgrade.openeuler.org/node-selector=kubeos的label的worker节点进行升级 ++ * nodeselector对配置和回滚同样有效 ++ * 节点添加label和label修改命令示例如下： ++ ``` shell ++ # 为节点kubeos-node1增加label ++ kubectl label nodes kubeos-node1 upgrade.openeuler.org/node-selector=kubeos-v1 ++ # 修改节点kubeos-node1的label ++ kubectl label --overwrite nodes kubeos-node2 upgrade.openeuler.org/node-selector=kubeos-v2 + ++ ``` ++ * yaml示例如下： ++ ```yaml ++ apiVersion: upgrade.openeuler.org/v1alpha1 ++ kind: OS ++ metadata: ++ name: os-sample ++ spec: ++ imagetype: containerd ++ opstype: upgrade ++ osversion: edit.os.version ++ maxunavailable: edit.node.upgrade.number ++ containerimage: container image like repository/name:tag ++ evictpodforce: true/false ++ imageurl: "" ++ checksum: container image digests ++ flagSafe: false ++ mtls: true ++ nodeselector: edit.node.label.key ++ ``` + * 查看未升级的节点的 OS 版本 + + ```shell +@@ -711,8 +741,7 @@ hostshell + operation: delete + ``` + +-* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: +- ++* kernel.sysctl.persist: 设置持久化内核参数，key/value表示内核参数的key/value，key与value均不能为空且key不能包含“=”， configpath为配置文件路径，支持新建（需保证父目录存在），如不指定configpath默认修改/etc/sysctl.conf，示例如下： + ```yaml + configs: + - model: kernel.sysctl.persist +diff --git a/pkg/values/values.go b/pkg/values/values.go +index f488ae52..e2c03540 100644 +--- a/pkg/values/values.go ++++ b/pkg/values/values.go +@@ -26,7 +26,11 @@ const ( + LabelMaster = "node-role.kubernetes.io/control-plane" + // LabelOSinstance is used to select the osinstance with the nodeName by label + LabelOSinstance = "upgrade.openeuler.org/osinstance-node" +- defaultPeriod = 15 * time.Second ++ // LabelNodeSelector is used to filter the nodes that need to be upgraded or configured. ++ LabelNodeSelector = "upgrade.openeuler.org/node-selector" ++ // LabelConfiguring is the key of the configuring label for nodes ++ LabelConfiguring = "upgrade.openeuler.org/configuring" ++ defaultPeriod = 15 * time.Second + // OsiStatusName is param name of nodeStatus in osInstance + OsiStatusName = "nodestatus" + // UpgradeConfigName is param name of UpgradeConfig +-- +2.33.0.windows.2 + diff --git a/0005-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch b/0005-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch new file mode 100644 index 0000000..52ad34b --- /dev/null +++ b/0005-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch @@ -0,0 +1,33 @@ +From 5dda90bb33e8f914bdc32f1f910786a969cffbae Mon Sep 17 00:00:00 2001 +From: Yuhang Wei +Date: Fri, 24 Nov 2023 10:54:21 +0800 +Subject: [PATCH 5/9] fix(agent, proxy): transform log timestamp to + human-readable format + +Originally, the log of controllers is timestamp which is hard to read. Now, transform the log into more human-readable format. + +Signed-off-by: Yuhang Wei +--- + cmd/operator/controllers/operation.go | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go +index 4b441d18..106892c4 100644 +--- a/cmd/operator/controllers/operation.go ++++ b/cmd/operator/controllers/operation.go +@@ -20,10 +20,11 @@ import ( + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" +- upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" + "sigs.k8s.io/controller-runtime/pkg/client" ++ ++ upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + ) + + type operation interface { +-- +2.33.0.windows.2 + diff --git a/0006-operator-delete-unnecessary-fmt-and-add-printing-for.patch b/0006-operator-delete-unnecessary-fmt-and-add-printing-for.patch new file mode 100644 index 0000000..0b12516 --- /dev/null +++ b/0006-operator-delete-unnecessary-fmt-and-add-printing-for.patch @@ -0,0 +1,137 @@ +From a9c9847ca824c24acdd1f93d19374f45a38f28d1 Mon Sep 17 00:00:00 2001 +From: liyuanr +Date: Fri, 24 Nov 2023 10:52:01 +0800 +Subject: [PATCH 6/9] operator: delete unnecessary fmt and add printing for + upgrades and configurations + +Delete unnecessary fmts, update osinstance during upgrades and configurations, and add printing of node labels + +Signed-off-by: liyuanr +--- + cmd/operator/controllers/operation.go | 6 ++++++ + cmd/operator/controllers/os_controller.go | 14 -------------- + 2 files changed, 6 insertions(+), 14 deletions(-) + +diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go +index 106892c4..9c9c6426 100644 +--- a/cmd/operator/controllers/operation.go ++++ b/cmd/operator/controllers/operation.go +@@ -65,6 +65,7 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, + } + osVersionNode := node.Status.NodeInfo.OSImage + if os.Spec.OSVersion != osVersionNode { ++ log.Info("Upgrading node " + node.Name) + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { +@@ -109,11 +110,13 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } ++ log.Info("Update osinstance spec successfully") + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } ++ log.Info("Add node upgrading label " + values.LabelUpgrading + " successfully") + return nil + } + +@@ -153,6 +156,7 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o + continue + } + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { ++ log.Info("Configuring node " + node.Name) + if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue +@@ -173,10 +177,12 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } ++ log.Info("Update osinstance spec successfully") + node.Labels[values.LabelConfiguring] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } ++ log.Info("Add node configuring label " + values.LabelConfiguring + " successfully") + return nil + } +diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go +index bd7a70dd..2c4ee1f7 100644 +--- a/cmd/operator/controllers/os_controller.go ++++ b/cmd/operator/controllers/os_controller.go +@@ -17,7 +17,6 @@ import ( + "context" + "encoding/json" + "fmt" +- "reflect" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" +@@ -159,10 +158,7 @@ func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.N + + func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + ops operation) (bool, error) { +- fmt.Println("start assignOperation") +- fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newNotExistRequirement() +- fmt.Println("requirement is ", requirement.String()) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return false, err +@@ -180,7 +176,6 @@ func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return false, err + } +- fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) + } + +@@ -188,7 +183,6 @@ func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev + if err != nil { + return false, err + } +- fmt.Println("nodes has not upgrade/config and has selector ", len(nodes)) + // Upgrade OS for selected nodes + count, err := ops.updateNodes(ctx, r, &os, nodes, limit) + if err != nil { +@@ -211,14 +205,11 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, + + func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + ops operation, maxUnavailable int, nodeSelector string) (int, error) { +- fmt.Println("start calNodeLimit") +- fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newExistRequirement() + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return 0, err + } +- fmt.Println("requirement is ", requirement.String()) + var requirements []labels.Requirement + requirements = append(requirements, requirement) + if nodeSelector != "" { +@@ -227,7 +218,6 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return 0, err + } +- fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) + } + nodes, err := getNodes(ctx, r, 0, requirements...) +@@ -235,10 +225,6 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + return 0, err + + } +- fmt.Println("nodes has upgrade and selector ", len(nodes)) +- for _, n := range nodes { +- fmt.Println(" nodes name is ", n.Name) +- } + return maxUnavailable - len(nodes), nil + } + +-- +2.33.0.windows.2 + diff --git a/0007-feat-os-operator-support-setting-TimeWindow-and-Time.patch b/0007-feat-os-operator-support-setting-TimeWindow-and-Time.patch new file mode 100644 index 0000000..464887d --- /dev/null +++ b/0007-feat-os-operator-support-setting-TimeWindow-and-Time.patch @@ -0,0 +1,628 @@ +From 9f3e6900d23918907298f8b8646ef6faf8f22e3c Mon Sep 17 00:00:00 2001 +From: liyuanr +Date: Wed, 14 Aug 2024 21:01:42 +0800 +Subject: [PATCH 7/9] feat(os-operator): support setting TimeWindow and + TimeInterval + +The timewindow and timeInterval parameters are added to the OS CRD. +The timewindow and timeinterval can be set during upgrade or configuration. + +Signed-off-by: liyuanr +--- + api/v1alpha1/os_types.go | 10 ++ + cmd/operator/controllers/operation.go | 45 +---- + cmd/operator/controllers/os_controller.go | 210 ++++++++++++++-------- + cmd/operator/controllers/requirements.go | 85 +++++++++ + cmd/operator/controllers/times.go | 100 +++++++++++ + 5 files changed, 341 insertions(+), 109 deletions(-) + create mode 100644 cmd/operator/controllers/requirements.go + create mode 100644 cmd/operator/controllers/times.go + +diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go +index d3d636de..65bb8e2e 100644 +--- a/api/v1alpha1/os_types.go ++++ b/api/v1alpha1/os_types.go +@@ -39,7 +39,12 @@ type OSSpec struct { + // +kubebuilder:validation:Optional + UpgradeConfigs SysConfigs `json:"upgradeconfigs"` + // +kubebuilder:validation:Optional ++ // +kubebuilder:default:=no-label + NodeSelector string `json:"nodeselector"` ++ // +kubebuilder:validation:Optional ++ TimeWindow TimeWindow `json:"timewindow"` ++ // +kubebuilder:validation:Optional ++ TimeInterval int `json:"timeinterval"` + } + + // +kubebuilder:subresource:status +@@ -90,6 +95,11 @@ type Content struct { + Operation string `json:"operation"` + } + ++type TimeWindow struct { ++ StartTime string `json:"starttime"` ++ EndTime string `json:"endtime"` ++} ++ + // +kubebuilder:subresource:status + // +kubebuilder:object:root=true + +diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go +index 9c9c6426..2ffcaa2e 100644 +--- a/cmd/operator/controllers/operation.go ++++ b/cmd/operator/controllers/operation.go +@@ -17,8 +17,6 @@ import ( + "context" + + corev1 "k8s.io/api/core/v1" +- "k8s.io/apimachinery/pkg/labels" +- "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" +@@ -28,32 +26,19 @@ import ( + ) + + type operation interface { +- newExistRequirement() (labels.Requirement, error) +- newNotExistRequirement() (labels.Requirement, error) + updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) + updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error ++ getOpsLabel() opsLabel + } + +-type upgradeOps struct{} +- +-func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { +- requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelUpgrading) +- return labels.Requirement{}, err +- } +- return *requirement, nil ++type upgradeOps struct { ++ label opsLabel + } + +-func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { +- requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelUpgrading) +- return labels.Requirement{}, err +- } +- return *requirement, nil ++func (u upgradeOps) getOpsLabel() opsLabel { ++ return u.label + } + + func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +@@ -120,24 +105,12 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW + return nil + } + +-type configOps struct{} +- +-func (c configOps) newExistRequirement() (labels.Requirement, error) { +- requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelConfiguring) +- return labels.Requirement{}, err +- } +- return *requirement, nil ++type configOps struct { ++ label opsLabel + } + +-func (c configOps) newNotExistRequirement() (labels.Requirement, error) { +- requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelConfiguring) +- return labels.Requirement{}, err +- } +- return *requirement, nil ++func (c configOps) getOpsLabel() opsLabel { ++ return c.label + } + + func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go +index 2c4ee1f7..690b6458 100644 +--- a/cmd/operator/controllers/os_controller.go ++++ b/cmd/operator/controllers/os_controller.go +@@ -17,6 +17,7 @@ import ( + "context" + "encoding/json" + "fmt" ++ "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" +@@ -56,7 +57,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re + + // Reconcile compares the actual state with the desired and updates the status of the resources e.g. nodes + func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) (ctrl.Result, error) { +- os, nodeNum, err := getAndUpdateOS(ctx, r, req.NamespacedName) ++ os, err := getOSCr(ctx, r, req.NamespacedName) + if err != nil { + if errors.IsNotFound(err) { + return values.NoRequeue, nil +@@ -64,18 +65,31 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) + return values.RequeueNow, err + } + ++ isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) ++ if err != nil { ++ return values.RequeueNow, err ++ } ++ if !isWithinTimeWindow { ++ //Todo consider time interval ++ return values.RequeueNow, nil ++ } ++ + ops := os.Spec.OpsType + var opsInsatnce operation + switch ops { + case "upgrade", "rollback": +- opsInsatnce = upgradeOps{} ++ opsInsatnce = upgradeOps{label: opsLabel{label: values.LabelUpgrading, op: selection.DoesNotExist}} + case "config": +- opsInsatnce = configOps{} ++ opsInsatnce = configOps{label: opsLabel{label: values.LabelConfiguring, op: selection.DoesNotExist}} + default: + log.Error(nil, "operation "+ops+" cannot be recognized") + return values.Requeue, nil + } +- limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need ++ nodeNum, err := getNodeNum(ctx, r, os.Spec.NodeSelector) ++ if err != nil { ++ return values.RequeueNow, err ++ } ++ limit, err := calNodeLimit(ctx, r, opsInsatnce.getOpsLabel(), min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + if err != nil { + return values.RequeueNow, err + } +@@ -84,7 +98,7 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) + } else if needRequeue { + return values.Requeue, nil + } +- return values.Requeue, nil ++ return setTimeInterval(os.Spec.TimeInterval), nil + } + + // SetupWithManager sets up the controller with the Manager. +@@ -100,6 +114,28 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { + }); err != nil { + return err + } ++ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", ++ func(rawObj client.Object) []string { ++ os, ok := rawObj.(*upgradev1.OS) ++ if !ok { ++ log.Error(nil, "failed to convert to osInstance") ++ return []string{} ++ } ++ return []string{os.Name} ++ }); err != nil { ++ return err ++ } ++ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", ++ func(rawObj client.Object) []string { ++ os, ok := rawObj.(*upgradev1.OS) ++ if !ok { ++ log.Error(nil, "failed to convert to osInstance") ++ return []string{} ++ } ++ return []string{os.Namespace} ++ }); err != nil { ++ return err ++ } + return ctrl.NewControllerManagedBy(mgr). + For(&upgradev1.OS{}). + Watches(&source.Kind{Type: &corev1.Node{}}, handler.Funcs{DeleteFunc: r.DeleteOSInstance}). +@@ -124,72 +160,33 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim + } + } + +-func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, +- int, error) { ++func getOSCr(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, error) { + var os upgradev1.OS + if err := r.Get(ctx, name, &os); err != nil { + log.Error(err, "unable to fetch OS") +- return upgradev1.OS{}, 0, err ++ return upgradev1.OS{}, err ++ } ++ if err := checkNodeSelector(ctx, r, os); err != nil { ++ log.Error(err, "nodeselector conficts") ++ return upgradev1.OS{}, err + } ++ return os, nil ++} + +- requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) ++// Get nodes which do not have master label，have nodeselector label or do not have other os cr nodeselector ++func getNodeNum(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) (int, error) { ++ labelList := []labelRequirementCreator{masterLabel{op: selection.DoesNotExist}, nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} ++ requirements, err := createRequirement(labelList) + if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelMaster) +- return upgradev1.OS{}, 0, err +- } +- var requirements []labels.Requirement +- requirements = append(requirements, *requirement) +- if os.Spec.NodeSelector != "" { +- reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelNodeSelector) +- return upgradev1.OS{}, 0, err +- } +- requirements = append(requirements, *requirement, *reqSelector) ++ return 0, err + } + nodesItems, err := getNodes(ctx, r, 0, requirements...) + if err != nil { + log.Error(err, "get slave nodes fail") +- return upgradev1.OS{}, 0, err ++ return 0, err + } + nodeNum := len(nodesItems) +- return os, nodeNum, nil +-} +- +-func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, +- ops operation) (bool, error) { +- requirement, err := ops.newNotExistRequirement() +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelUpgrading) +- return false, err +- } +- reqMaster, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelMaster) +- return false, err +- } +- var requirements []labels.Requirement +- requirements = append(requirements, requirement, *reqMaster) +- if os.Spec.NodeSelector != "" { +- reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelNodeSelector) +- return false, err +- } +- requirements = append(requirements, *reqSelector) +- } +- +- nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated +- if err != nil { +- return false, err +- } +- // Upgrade OS for selected nodes +- count, err := ops.updateNodes(ctx, r, &os, nodes, limit) +- if err != nil { +- return false, err +- } +- +- return count >= limit, nil ++ return nodeNum, nil + } + + func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, +@@ -203,31 +200,47 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, + return nodeList.Items, nil + } + ++// get now in upgrading and match with nodeselector + func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, +- ops operation, maxUnavailable int, nodeSelector string) (int, error) { +- requirement, err := ops.newExistRequirement() ++ label opsLabel, maxUnavailable int, nodeSelector string) (int, error) { ++ label.op = selection.Exists ++ labelList := []labelRequirementCreator{ ++ masterLabel{op: selection.DoesNotExist}, ++ label, ++ nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} ++ requirements, err := createRequirement(labelList) + if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return 0, err + } +- var requirements []labels.Requirement +- requirements = append(requirements, requirement) +- if nodeSelector != "" { +- reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelNodeSelector) +- return 0, err +- } +- requirements = append(requirements, *reqSelector) +- } + nodes, err := getNodes(ctx, r, 0, requirements...) + if err != nil { + return 0, err +- + } + return maxUnavailable - len(nodes), nil + } + ++func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, ++ opsInstance operation) (bool, error) { ++ opsLabel := opsInstance.getOpsLabel() ++ opsLabel.op = selection.DoesNotExist ++ labelList := []labelRequirementCreator{ ++ masterLabel{op: selection.DoesNotExist}, ++ opsLabel, ++ nodeSelectorLabel{value: os.Spec.NodeSelector, op: selection.Equals}} ++ requirements, err := createRequirement(labelList) ++ nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated ++ if err != nil { ++ return false, err ++ } ++ // Upgrade OS for selected nodes ++ count, err := opsInstance.updateNodes(ctx, r, &os, nodes, limit) ++ if err != nil { ++ return false, err ++ } ++ ++ return count >= limit, nil ++} ++ + func min(a, b int) int { + if a < b { + return a +@@ -259,3 +272,54 @@ func deepCopySpecConfigs(os *upgradev1.OS, osinstance *upgradev1.OSInstance, con + } + return nil + } ++ ++// Check whether the nodeselector conflicts with other nodeselector in OS CRs. If the nodeselector is empty, return the list of other nodeselectors. ++func checkNodeSelector(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS) error { ++ var osList upgradev1.OSList ++ if err := r.List(ctx, &osList, &client.ListOptions{}); err != nil { ++ log.Error(err, "unable to list nodes with requirements") ++ return err ++ } ++ var sameNodeSelectorList []types.NamespacedName ++ for _, osItem := range osList.Items { ++ // Exclude current os, controller-runtime not supports multiple indexs as listoptions in current version, ++ // so cannot list os without current os use List function ++ if osItem.Name == os.Name && osItem.Namespace == os.Namespace { ++ continue ++ } ++ if os.Spec.NodeSelector == osItem.Spec.NodeSelector { ++ sameNodeSelectorList = append(sameNodeSelectorList, types.NamespacedName{ ++ Namespace: osItem.Namespace, ++ Name: osItem.Name, ++ }) ++ } ++ } ++ // If a node label corresponds to multiple OS CRs, upgrade or configuration information may conflict. ++ // As a result, an error is reported and returned when there are one-to-many relationships. ++ if len(sameNodeSelectorList) > 0 { ++ errorMessage := sameNodeSelectorList[0].String() ++ for i := 1; i < len(sameNodeSelectorList); i++ { ++ errorMessage = errorMessage + " , " + sameNodeSelectorList[i].String() ++ } ++ log.Error(nil, "OS CR "+os.Name+" in namespace "+os.Namespace+" has same nodeselector with "+errorMessage) ++ return fmt.Errorf("OS CR %s in namespace %s has same nodeselector with %s", os.Name, os.Namespace, errorMessage) ++ } ++ return nil ++} ++ ++func setTimeInterval(timeInterval int) ctrl.Result { ++ return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(timeInterval) * time.Second} ++} ++ ++func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { ++ var requirements []labels.Requirement ++ for _, label := range labelsList { ++ requirement, err := label.createLabelRequirement() ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return []labels.Requirement{}, err ++ } ++ requirements = append(requirements, requirement...) ++ } ++ return requirements, nil ++} +diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go +new file mode 100644 +index 00000000..5a8f255f +--- /dev/null ++++ b/cmd/operator/controllers/requirements.go +@@ -0,0 +1,85 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * KubeOS is licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ */ ++ ++// Package controllers contains the Reconcile of operator ++package controllers ++ ++import ( ++ "k8s.io/apimachinery/pkg/labels" ++ "k8s.io/apimachinery/pkg/selection" ++ "openeuler.org/KubeOS/pkg/values" ++) ++ ++const ( ++ AllNodeSelector = "all-label" ++ NoNodeSelector = "no-label" ++) ++ ++type labelRequirementCreator interface { ++ createLabelRequirement() ([]labels.Requirement, error) ++} ++ ++type masterLabel struct { ++ op selection.Operator ++} ++ ++func (ml masterLabel) createLabelRequirement() ([]labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(values.LabelMaster, ml.op, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelMaster) ++ return []labels.Requirement{}, err ++ } ++ return []labels.Requirement{*requirement}, nil ++} ++ ++type opsLabel struct { ++ label string ++ op selection.Operator ++} ++ ++func (ol opsLabel) createLabelRequirement() ([]labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(ol.label, ol.op, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+ol.label) ++ return []labels.Requirement{}, err ++ } ++ return []labels.Requirement{*requirement}, nil ++} ++ ++type nodeSelectorLabel struct { ++ value string ++ op selection.Operator ++} ++ ++func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, error) { ++ if nl.value == AllNodeSelector { ++ return []labels.Requirement{}, nil ++ } ++ var requirements []labels.Requirement ++ // if nodeselector is "", will get the nodes which label value is "",and not have label ++ if nl.value == NoNodeSelector { ++ requirement, err := labels.NewRequirement(values.LabelNodeSelector, selection.DoesNotExist, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return []labels.Requirement{}, err ++ } ++ requirements = append(requirements, *requirement) ++ return requirements, nil ++ } ++ requirement, err := labels.NewRequirement(values.LabelNodeSelector, nl.op, []string{nl.value}) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return []labels.Requirement{}, err ++ } ++ requirements = append(requirements, *requirement) ++ return requirements, nil ++} +diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go +new file mode 100644 +index 00000000..eab3a224 +--- /dev/null ++++ b/cmd/operator/controllers/times.go +@@ -0,0 +1,100 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * KubeOS is licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ */ ++ ++// Package controllers contains the Reconcile of operator ++ ++package controllers ++ ++import ( ++ "fmt" ++ "regexp" ++ "time" ++) ++ ++const ( ++ DATE_TIME = "2006-01-02 15:04:05" ++ TIME_ONLY = "15:04:05" ++ ExecutionModeSerial = "serial" ++ ExecutionModeParallel = "parallel" ++) ++ ++func isWithinTimeWindow(start, end string) (bool, error) { ++ if start == "" && end == "" { ++ return true, nil ++ } ++ if start == "" || end == "" { ++ return false, fmt.Errorf("The start time and end time must be both empty or not empty.") ++ } ++ layoutStart, err := checkTimeValid(start) ++ if err != nil { ++ return false, err ++ } ++ layoutEnd, err := checkTimeValid(end) ++ if err != nil { ++ return false, err ++ } ++ if layoutStart != layoutEnd { ++ return false, fmt.Errorf("Start Time needs same time format with End Time") ++ } ++ now := time.Now() ++ if layoutStart == TIME_ONLY { ++ timeFormat := now.Format(TIME_ONLY) ++ now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) ++ } ++ startTime, err := time.ParseInLocation(layoutStart, start, now.Location()) ++ if err != nil { ++ return false, err ++ } ++ endTime, err := time.ParseInLocation(layoutStart, end, now.Location()) ++ if err != nil { ++ return false, err ++ } ++ if endTime.Before(startTime) { ++ endTime = endTime.Add(24 * time.Hour) ++ fmt.Printf("endtime time add 24 hour is %s\n", endTime.Format(layoutStart)) ++ if now.Before(startTime) { ++ now = now.Add(24 * time.Hour) ++ fmt.Printf("now time add 24 hour is %s\n", now.Format(layoutStart)) ++ } ++ ++ } ++ return now.After(startTime) && now.Before(endTime), nil ++} ++ ++func checkTimeValid(checkTime string) (string, error) { ++ reDateTime, err := regexp.Compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$") ++ if err != nil { ++ return "", err ++ } ++ reTimeOnly, err := regexp.Compile("^\\d{2}:\\d{2}:\\d{2}$") ++ if err != nil { ++ return "", err ++ } ++ ++ if reDateTime.MatchString(checkTime) { ++ _, err := time.Parse(DATE_TIME, checkTime) ++ if err != nil { ++ return "", err ++ } ++ return DATE_TIME, nil ++ ++ } ++ if reTimeOnly.MatchString(checkTime) { ++ _, err := time.Parse(TIME_ONLY, checkTime) ++ if err != nil { ++ return "", err ++ } ++ return TIME_ONLY, nil ++ ++ } ++ return "", fmt.Errorf("Invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS such as \"2006-01-02 15:04:05\" or \"15:04:05\"") ++} +-- +2.33.0.windows.2 + diff --git a/0008-feat-os-proxy-add-ExcutionMode-to-os.patch b/0008-feat-os-proxy-add-ExcutionMode-to-os.patch new file mode 100644 index 0000000..bf4d140 --- /dev/null +++ b/0008-feat-os-proxy-add-ExcutionMode-to-os.patch @@ -0,0 +1,865 @@ +From 6b5c8bf60875303e86817b086829b3934550388b Mon Sep 17 00:00:00 2001 +From: liyuanr +Date: Sat, 17 Aug 2024 19:37:10 +0800 +Subject: [PATCH 8/9] feat(os-proxy): add ExcutionMode to os + +add ExcutionMode to os to support serial during upgrade or configuration. + +Signed-off-by: liyuanr +--- + .../proxy/src/controller/controller.rs | 6 + + KubeOS-Rust/proxy/src/controller/values.rs | 1 + + api/v1alpha1/os_types.go | 10 +- + cmd/operator/controllers/operation.go | 141 ++++++++++- + cmd/operator/controllers/os_controller.go | 220 +++++++++++------- + cmd/operator/controllers/requirements.go | 115 ++++++++- + cmd/operator/main.go | 13 +- + .../config/crd/upgrade.openeuler.org_os.yaml | 27 +++ + pkg/values/values.go | 5 +- + 9 files changed, 436 insertions(+), 102 deletions(-) + +diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs +index 80a85d1c..baf2f9d8 100644 +--- a/KubeOS-Rust/proxy/src/controller/controller.rs ++++ b/KubeOS-Rust/proxy/src/controller/controller.rs +@@ -24,6 +24,8 @@ use kube::{ + use log::{debug, error, info}; + use reconciler_error::Error; + ++use crate::controller::values::LABEL_CONFIGURING; ++ + use super::{ + agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + apiclient::ApplyApi, +@@ -188,6 +190,10 @@ impl ProxyController { + if labels.contains_key(LABEL_UPGRADING) { + labels.remove(LABEL_UPGRADING); + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; ++ }else if labels.contains_key(LABEL_CONFIGURING) { ++ labels.remove(LABEL_CONFIGURING); ++ node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; ++ + } + if let Some(node_spec) = &node.spec { + if let Some(node_unschedulable) = node_spec.unschedulable { +diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs +index dec905a9..9066f57d 100644 +--- a/KubeOS-Rust/proxy/src/controller/values.rs ++++ b/KubeOS-Rust/proxy/src/controller/values.rs +@@ -15,6 +15,7 @@ use tokio::time::Duration; + + pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; + pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; ++pub const LABEL_CONFIGURING: &str = "upgrade.openeuler.org/configuring"; + + pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; + pub const OSINSTANCE_KIND: &str = "OSInstance"; +diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go +index 65bb8e2e..6439bc14 100644 +--- a/api/v1alpha1/os_types.go ++++ b/api/v1alpha1/os_types.go +@@ -24,10 +24,12 @@ type OSSpec struct { + CheckSum string `json:"checksum"` + FlagSafe bool `json:"flagSafe"` + MTLS bool `json:"mtls"` ++ // +kubebuilder:validation:Enum=docker;disk;containerd + ImageType string `json:"imagetype"` + ContainerImage string `json:"containerimage"` +- OpsType string `json:"opstype"` +- EvictPodForce bool `json:"evictpodforce"` ++ // +kubebuilder:validation:Enum=upgrade;config;rollback ++ OpsType string `json:"opstype"` ++ EvictPodForce bool `json:"evictpodforce"` + // +kubebuilder:validation:Optional + CaCert string `json:"cacert"` + // +kubebuilder:validation:Optional +@@ -45,6 +47,10 @@ type OSSpec struct { + TimeWindow TimeWindow `json:"timewindow"` + // +kubebuilder:validation:Optional + TimeInterval int `json:"timeinterval"` ++ // +kubebuilder:validation:Optional ++ // +kubebuilder:validation:Enum=serial;parallel ++ // +kubebuilder:default:=parallel ++ ExecutionMode string `json:"executionmode"` + } + + // +kubebuilder:subresource:status +diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go +index 2ffcaa2e..dd0b4fec 100644 +--- a/cmd/operator/controllers/operation.go ++++ b/cmd/operator/controllers/operation.go +@@ -15,19 +15,20 @@ package controllers + + import ( + "context" ++ "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +- "openeuler.org/KubeOS/pkg/common" +- "openeuler.org/KubeOS/pkg/values" + "sigs.k8s.io/controller-runtime/pkg/client" + + upgradev1 "openeuler.org/KubeOS/api/v1alpha1" ++ "openeuler.org/KubeOS/pkg/common" ++ "openeuler.org/KubeOS/pkg/values" + ) + + type operation interface { + updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +- nodes []corev1.Node, limit int) (int, error) ++ nodes []corev1.Node, limit int) (int, []error) + updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error + getOpsLabel() opsLabel +@@ -42,8 +43,9 @@ func (u upgradeOps) getOpsLabel() opsLabel { + } + + func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +- nodes []corev1.Node, limit int) (int, error) { +- var count int ++ nodes []corev1.Node, limit int) (int, []error) { ++ var count = 0 ++ var errList []error + for _, node := range nodes { + if count >= limit { + break +@@ -54,19 +56,49 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { +- log.Error(err, "failed to get osInstance "+node.Name, "skip this node") +- return count, err ++ log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ ++ "Restart the reconcile and wait until it is complete.") ++ return count, []error{err} + } ++ log.Error(err, "failed to get osInstance "+node.Name+"skip this node") ++ errList = append(errList, err) + continue + } + if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") ++ errList = append(errList, err) + continue + } + count++ + } + } ++ if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { ++ if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { ++ log.Error(nil, "failed to delete nodes serial label") ++ } ++ } ++ if len(errList) > 0 { ++ return count, errList ++ } + return count, nil ++ ++} ++ ++func deleteSerialLabel(ctx context.Context, r common.ReadStatusWriter, nodes []corev1.Node) []error { ++ var errList []error ++ for _, node := range nodes { ++ if _, ok := node.Labels[values.LabelSerial]; ok { ++ delete(node.Labels, values.LabelSerial) ++ if err := r.Update(ctx, &node); err != nil { ++ log.Error(err, "unable to delete serial label ", "node", node.Name+", skip this node") ++ errList = append(errList, err) ++ } ++ } ++ } ++ if len(errList) > 0 { ++ return errList ++ } ++ return nil + } + + func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +@@ -114,8 +146,9 @@ func (c configOps) getOpsLabel() opsLabel { + } + + func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +- nodes []corev1.Node, limit int) (int, error) { +- var count int ++ nodes []corev1.Node, limit int) (int, []error) { ++ var count = 0 ++ var errList []error + for _, node := range nodes { + if count >= limit { + break +@@ -123,21 +156,34 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { +- log.Error(err, "failed to get osInstance "+node.Name) +- return count, err ++ log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ ++ "Restart the reconcile and wait until it is complete.") ++ return count, []error{err} + } ++ log.Error(err, "failed to get osInstance "+node.Name+", skip this node") ++ errList = append(errList, err) + continue + } + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + log.Info("Configuring node " + node.Name) + if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") ++ errList = append(errList, err) + continue + } + count++ + } + } +- return count, nil ++ if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { ++ if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { ++ log.Error(nil, "failed to delete nodes serial label") ++ } ++ } ++ if len(errList) > 0 { ++ return count, errList ++ } ++ return count, errList ++ + } + + func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, +@@ -159,3 +205,74 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr + log.Info("Add node configuring label " + values.LabelConfiguring + " successfully") + return nil + } ++ ++type serialOps struct { ++ label opsLabel ++} ++ ++func (s serialOps) getOpsLabel() opsLabel { ++ return s.label ++} ++ ++func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ nodes []corev1.Node, limit int) (int, []error) { ++ var count int ++ var errList []error ++ for _, node := range nodes { ++ log.V(1).Info("start add serial label to nodes") ++ if count >= limit { ++ break ++ } ++ var osInstance upgradev1.OSInstance ++ if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { ++ if err = client.IgnoreNotFound(err); err != nil { ++ log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ ++ "Restart the reconcile and wait until it is complete.") ++ return count, []error{err} ++ } ++ log.Error(err, "failed to get osInstance "+node.Name+", skip this node") ++ errList = append(errList, err) ++ continue ++ } ++ switch s.getOpsLabel().label { ++ case values.LabelUpgrading: ++ if os.Spec.OSVersion != node.Status.NodeInfo.OSImage { ++ log.Info("Add Serial Label to node " + node.Name) ++ if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { ++ log.Error(err, "failed to update node and osinstance ,skip this node ") ++ errList = append(errList, err) ++ continue ++ } ++ count++ ++ } ++ case values.LabelConfiguring: ++ if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { ++ log.Info("Add Serial Label to node " + node.Name) ++ if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { ++ log.Error(err, "failed to update node and osinstance ,skip this node ") ++ errList = append(errList, err) ++ continue ++ } ++ count++ ++ } ++ default: ++ log.Error(nil, "ops "+s.getOpsLabel().label+" cannot be recognized") ++ return count, []error{fmt.Errorf("ops " + s.getOpsLabel().label + " cannot be recognized")} ++ } ++ } ++ if len(errList) == 0 { ++ return count, nil ++ } ++ return count, errList ++} ++func (s serialOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, ++ node *corev1.Node, osInstance *upgradev1.OSInstance) error { ++ log.V(1).Info("start update nodes") ++ node.Labels[values.LabelSerial] = "" ++ if err := r.Update(ctx, node); err != nil { ++ log.Error(err, "unable to label", "node", node.Name) ++ return err ++ } ++ log.Info("Add node serial label " + values.LabelSerial + " successfully") ++ return nil ++} +diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go +index 690b6458..667a265f 100644 +--- a/cmd/operator/controllers/os_controller.go ++++ b/cmd/operator/controllers/os_controller.go +@@ -17,6 +17,7 @@ import ( + "context" + "encoding/json" + "fmt" ++ "strconv" + "time" + + corev1 "k8s.io/api/core/v1" +@@ -57,6 +58,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re + + // Reconcile compares the actual state with the desired and updates the status of the resources e.g. nodes + func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) (ctrl.Result, error) { ++ log.V(1).Info("start Reconcile of " + req.Name + " with namespace is " + req.Namespace) + os, err := getOSCr(ctx, r, req.NamespacedName) + if err != nil { + if errors.IsNotFound(err) { +@@ -64,41 +66,64 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) + } + return values.RequeueNow, err + } +- ++ log.V(1).Info("get os cr name is " + os.Name + ", namespace is " + os.Namespace) + isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) + if err != nil { + return values.RequeueNow, err + } + if !isWithinTimeWindow { +- //Todo consider time interval +- return values.RequeueNow, nil ++ log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + " , the start time " + os.Spec.TimeWindow.StartTime) ++ return values.Requeue, nil + } + + ops := os.Spec.OpsType + var opsInsatnce operation + switch ops { + case "upgrade", "rollback": +- opsInsatnce = upgradeOps{label: opsLabel{label: values.LabelUpgrading, op: selection.DoesNotExist}} ++ opsInsatnce = upgradeOps{ ++ label: opsLabel{ ++ label: values.LabelUpgrading, ++ op: selection.DoesNotExist, ++ }, ++ } + case "config": +- opsInsatnce = configOps{label: opsLabel{label: values.LabelConfiguring, op: selection.DoesNotExist}} ++ opsInsatnce = configOps{ ++ label: opsLabel{ ++ label: values.LabelConfiguring, ++ op: selection.DoesNotExist, ++ }, ++ } + default: + log.Error(nil, "operation "+ops+" cannot be recognized") + return values.Requeue, nil + } +- nodeNum, err := getNodeNum(ctx, r, os.Spec.NodeSelector) ++ commonNodesReq, err := newCommonsNodesRequirement(os.Spec.NodeSelector, ++ selection.Equals).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, err + } +- limit, err := calNodeLimit(ctx, r, opsInsatnce.getOpsLabel(), min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need ++ allNodes, err := getNodes(ctx, r, 0, commonNodesReq...) + if err != nil { + return values.RequeueNow, err + } +- if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { +- return values.RequeueNow, err +- } else if needRequeue { ++ log.V(1).Info("get all nodes is " + strconv.Itoa(len(allNodes))) ++ switch os.Spec.ExecutionMode { ++ case ExecutionModeParallel: ++ result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes)) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ return result, nil ++ case ExecutionModeSerial: ++ result, err := excuteSerialOperation(ctx, r, os, opsInsatnce, len(allNodes)) ++ if err != nil { ++ return values.RequeueNow, err ++ } ++ return result, nil ++ default: ++ log.Error(nil, "excutionMode "+os.Spec.ExecutionMode+" cannot be recognized") + return values.Requeue, nil + } +- return setTimeInterval(os.Spec.TimeInterval), nil + } + + // SetupWithManager sets up the controller with the Manager. +@@ -114,28 +139,28 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { + }); err != nil { + return err + } +- if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", +- func(rawObj client.Object) []string { +- os, ok := rawObj.(*upgradev1.OS) +- if !ok { +- log.Error(nil, "failed to convert to osInstance") +- return []string{} +- } +- return []string{os.Name} +- }); err != nil { +- return err +- } +- if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", +- func(rawObj client.Object) []string { +- os, ok := rawObj.(*upgradev1.OS) +- if !ok { +- log.Error(nil, "failed to convert to osInstance") +- return []string{} +- } +- return []string{os.Namespace} +- }); err != nil { +- return err +- } ++ // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", ++ // func(rawObj client.Object) []string { ++ // os, ok := rawObj.(*upgradev1.OS) ++ // if !ok { ++ // log.Error(nil, "failed to convert to osInstance") ++ // return []string{} ++ // } ++ // return []string{os.Name} ++ // }); err != nil { ++ // return err ++ // } ++ // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", ++ // func(rawObj client.Object) []string { ++ // os, ok := rawObj.(*upgradev1.OS) ++ // if !ok { ++ // log.Error(nil, "failed to convert to osInstance") ++ // return []string{} ++ // } ++ // return []string{os.Namespace} ++ // }); err != nil { ++ // return err ++ // } + return ctrl.NewControllerManagedBy(mgr). + For(&upgradev1.OS{}). + Watches(&source.Kind{Type: &corev1.Node{}}, handler.Funcs{DeleteFunc: r.DeleteOSInstance}). +@@ -173,22 +198,6 @@ func getOSCr(ctx context.Context, r common.ReadStatusWriter, name types.Namespac + return os, nil + } + +-// Get nodes which do not have master label，have nodeselector label or do not have other os cr nodeselector +-func getNodeNum(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) (int, error) { +- labelList := []labelRequirementCreator{masterLabel{op: selection.DoesNotExist}, nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} +- requirements, err := createRequirement(labelList) +- if err != nil { +- return 0, err +- } +- nodesItems, err := getNodes(ctx, r, 0, requirements...) +- if err != nil { +- log.Error(err, "get slave nodes fail") +- return 0, err +- } +- nodeNum := len(nodesItems) +- return nodeNum, nil +-} +- + func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, + reqs ...labels.Requirement) ([]corev1.Node, error) { + var nodeList corev1.NodeList +@@ -200,45 +209,27 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, + return nodeList.Items, nil + } + +-// get now in upgrading and match with nodeselector + func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, +- label opsLabel, maxUnavailable int, nodeSelector string) (int, error) { +- label.op = selection.Exists +- labelList := []labelRequirementCreator{ +- masterLabel{op: selection.DoesNotExist}, +- label, +- nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} +- requirements, err := createRequirement(labelList) +- if err != nil { +- return 0, err +- } ++ maxUnavailable int, requirements []labels.Requirement) (int, error) { + nodes, err := getNodes(ctx, r, 0, requirements...) + if err != nil { + return 0, err + } + return maxUnavailable - len(nodes), nil + } +- + func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, +- opsInstance operation) (bool, error) { +- opsLabel := opsInstance.getOpsLabel() +- opsLabel.op = selection.DoesNotExist +- labelList := []labelRequirementCreator{ +- masterLabel{op: selection.DoesNotExist}, +- opsLabel, +- nodeSelectorLabel{value: os.Spec.NodeSelector, op: selection.Equals}} +- requirements, err := createRequirement(labelList) ++ opsInstance operation, requirements []labels.Requirement) (int, error) { + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated + if err != nil { +- return false, err ++ return 0, err + } ++ log.V(1).Info("get need nodes is " + strconv.Itoa(len(nodes))) + // Upgrade OS for selected nodes +- count, err := opsInstance.updateNodes(ctx, r, &os, nodes, limit) +- if err != nil { +- return false, err ++ count, errLists := opsInstance.updateNodes(ctx, r, &os, nodes, limit) ++ if len(errLists) != 0 { ++ return 0, fmt.Errorf("update nodes and osinstance error") + } +- +- return count >= limit, nil ++ return count, nil + } + + func min(a, b int) int { +@@ -311,15 +302,74 @@ func setTimeInterval(timeInterval int) ctrl.Result { + return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(timeInterval) * time.Second} + } + +-func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { +- var requirements []labels.Requirement +- for _, label := range labelsList { +- requirement, err := label.createLabelRequirement() +- if err != nil { +- log.Error(err, "unable to create requirement "+values.LabelNodeSelector) +- return []labels.Requirement{}, err +- } +- requirements = append(requirements, requirement...) ++func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, ++ opsInsatnce operation, nodeNum int) (ctrl.Result, error) { ++ opsLabel := opsInsatnce.getOpsLabel() ++ opsLabel.op = selection.Exists ++ opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, ++ selection.Equals, opsLabel).createNodeRequirement(ctx, r) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ limit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), opsNodesReq) // adjust maxUnavailable if need ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ opsLabel.op = selection.DoesNotExist ++ noOpsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, ++ selection.Equals, opsLabel).createNodeRequirement(ctx, r) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ if _, err := assignOperation(ctx, r, os, limit, opsInsatnce, noOpsNodesReq); err != nil { ++ return values.RequeueNow, nil + } +- return requirements, nil ++ return setTimeInterval(os.Spec.TimeInterval), nil ++} ++ ++func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, ++ opsInsatnce operation, nodeNum int) (ctrl.Result, error) { ++ opsLabel := opsInsatnce.getOpsLabel() ++ opsLabel.op = selection.Exists ++ opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, ++ selection.Equals, opsLabel).createNodeRequirement(ctx, r) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ opsNodeNum, err := getNodes(ctx, r, 0, opsNodesReq...) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ if len(opsNodeNum) > 0 { ++ return values.Requeue, nil ++ } ++ log.V(1).Info("get opsnodes is " + strconv.Itoa(len(opsNodeNum))) ++ serialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, ++ selection.Equals, selection.Exists).createNodeRequirement(ctx, r) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ serialNodeLimit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), serialNodesRequirement) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ log.V(1).Info("get serialLimit is " + strconv.Itoa(serialNodeLimit)) ++ noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, ++ selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOps{opsInsatnce.getOpsLabel()}, noSerialNodesRequirement); err != nil { ++ return values.RequeueNow, nil ++ } ++ serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial ++ count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement) ++ if err != nil { ++ return values.RequeueNow, nil ++ } ++ if count > 0 { ++ return values.Requeue, nil ++ } ++ return setTimeInterval(os.Spec.TimeInterval), nil ++ + } +diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go +index 5a8f255f..9317b495 100644 +--- a/cmd/operator/controllers/requirements.go ++++ b/cmd/operator/controllers/requirements.go +@@ -14,8 +14,11 @@ + package controllers + + import ( ++ "context" ++ + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" ++ "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" + ) + +@@ -65,7 +68,7 @@ func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, erro + return []labels.Requirement{}, nil + } + var requirements []labels.Requirement +- // if nodeselector is "", will get the nodes which label value is "",and not have label ++ // if nodeselector is "no-label", will get the nodes which not have label + if nl.value == NoNodeSelector { + requirement, err := labels.NewRequirement(values.LabelNodeSelector, selection.DoesNotExist, nil) + if err != nil { +@@ -83,3 +86,113 @@ func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, erro + requirements = append(requirements, *requirement) + return requirements, nil + } ++ ++type serialLabel struct { ++ op selection.Operator ++} ++ ++func (sl serialLabel) createLabelRequirement() ([]labels.Requirement, error) { ++ requirement, err := labels.NewRequirement(values.LabelSerial, sl.op, nil) ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelSerial) ++ return []labels.Requirement{}, err ++ } ++ return []labels.Requirement{*requirement}, nil ++} ++ ++func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { ++ var requirements []labels.Requirement ++ for _, label := range labelsList { ++ requirement, err := label.createLabelRequirement() ++ if err != nil { ++ log.Error(err, "unable to create requirement "+values.LabelNodeSelector) ++ return []labels.Requirement{}, err ++ } ++ requirements = append(requirements, requirement...) ++ } ++ return requirements, nil ++} ++ ++type nodeRequirementCreator interface { ++ createNodeRequirement(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) ([]labels.Requirement, error) ++} ++ ++type commonsNodesRequirement struct { ++ nodeSelector string ++ op selection.Operator ++} ++ ++func newCommonsNodesRequirement(nodeSelector string, op selection.Operator) commonsNodesRequirement { ++ return commonsNodesRequirement{ ++ nodeSelector, ++ op, ++ } ++} ++ ++func (c commonsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { ++ labelList := []labelRequirementCreator{ ++ masterLabel{op: selection.DoesNotExist}, ++ nodeSelectorLabel{value: c.nodeSelector, op: c.op}, ++ } ++ requirements, err := createRequirement(labelList) ++ if err != nil { ++ return []labels.Requirement{}, err ++ } ++ return requirements, nil ++} ++ ++type opsNodesRequirement struct { ++ common commonsNodesRequirement ++ ops opsLabel ++} ++ ++func newopsNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, ops opsLabel) opsNodesRequirement { ++ return opsNodesRequirement{ ++ common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp), ++ ops: ops, ++ } ++} ++ ++func (o opsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { ++ labelList := []labelRequirementCreator{ ++ o.ops, ++ } ++ requirements, err := createRequirement(labelList) ++ if err != nil { ++ return []labels.Requirement{}, err ++ } ++ commonRequirements, err := o.common.createNodeRequirement(ctx, r) ++ if err != nil { ++ return []labels.Requirement{}, err ++ } ++ requirements = append(requirements, commonRequirements...) ++ return requirements, nil ++} ++ ++type serialNodesRequirement struct { ++ common commonsNodesRequirement ++ serialOp selection.Operator ++} ++ ++func newSerialNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, serialrOp selection.Operator) serialNodesRequirement { ++ return serialNodesRequirement{ ++ common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp), ++ serialOp: serialrOp, ++ } ++} ++ ++func (o serialNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { ++ labelList := []labelRequirementCreator{ ++ serialLabel{op: o.serialOp}, ++ } ++ requirements, err := createRequirement(labelList) ++ if err != nil { ++ return []labels.Requirement{}, err ++ } ++ commonRequirements, err := o.common.createNodeRequirement(ctx, r) ++ if err != nil { ++ return []labels.Requirement{}, err ++ } ++ requirements = append(requirements, commonRequirements...) ++ return requirements, nil ++} +diff --git a/cmd/operator/main.go b/cmd/operator/main.go +index 6b90b26b..79b23ec7 100644 +--- a/cmd/operator/main.go ++++ b/cmd/operator/main.go +@@ -14,6 +14,7 @@ package main + + import ( + "os" ++ "strings" + "time" + + zaplogfmt "github.com/sykesm/zap-logfmt" +@@ -38,6 +39,11 @@ var ( + setupLog = ctrl.Log.WithName("setup") + ) + ++const ( ++ DEBUGLOGLEVEL = "debug" ++ INFOLOGLEVEL = "info" ++) ++ + func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + +@@ -51,7 +57,12 @@ func main() { + encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) + } + logfmtEncoder := zaplogfmt.NewEncoder(configLog) +- logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder)) ++ level := zap.Level(zapcore.Level(0)) ++ goLog := strings.ToLower(os.Getenv("GO_LOG")) ++ if goLog == DEBUGLOGLEVEL { ++ level = zap.Level(zapcore.Level(-1)) ++ } ++ logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder), level) + ctrl.SetLogger(logger) + + mgr, err := common.NewControllerManager(setupLog, scheme) +diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml +index 27ff3273..f64bad64 100644 +--- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml ++++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml +@@ -57,9 +57,19 @@ spec: + type: string + evictpodforce: + type: boolean ++ executionmode: ++ default: parallel ++ enum: ++ - serial ++ - parallel ++ type: string + flagSafe: + type: boolean + imagetype: ++ enum: ++ - docker ++ - disk ++ - containerd + type: string + imageurl: + type: string +@@ -68,8 +78,13 @@ spec: + mtls: + type: boolean + nodeselector: ++ default: no-label + type: string + opstype: ++ enum: ++ - upgrade ++ - config ++ - rollback + type: string + osversion: + type: string +@@ -101,6 +116,18 @@ spec: + version: + type: string + type: object ++ timeinterval: ++ type: integer ++ timewindow: ++ properties: ++ endtime: ++ type: string ++ starttime: ++ type: string ++ required: ++ - endtime ++ - starttime ++ type: object + upgradeconfigs: + description: SysConfigs defines all configurations expected by the user + properties: +diff --git a/pkg/values/values.go b/pkg/values/values.go +index e2c03540..2045bbe6 100644 +--- a/pkg/values/values.go ++++ b/pkg/values/values.go +@@ -30,7 +30,10 @@ const ( + LabelNodeSelector = "upgrade.openeuler.org/node-selector" + // LabelConfiguring is the key of the configuring label for nodes + LabelConfiguring = "upgrade.openeuler.org/configuring" +- defaultPeriod = 15 * time.Second ++ // LabelSerial is the key of the serial label for nodes ++ LabelSerial = "upgrade.openeuler.org/serial" ++ ++ defaultPeriod = 15 * time.Second + // OsiStatusName is param name of nodeStatus in osInstance + OsiStatusName = "nodestatus" + // UpgradeConfigName is param name of UpgradeConfig +-- +2.33.0.windows.2 + diff --git a/0009-bugfix-fix-the-problem-that-proxy-will-get-all-os-fo.patch b/0009-bugfix-fix-the-problem-that-proxy-will-get-all-os-fo.patch new file mode 100644 index 0000000..443fffa --- /dev/null +++ b/0009-bugfix-fix-the-problem-that-proxy-will-get-all-os-fo.patch @@ -0,0 +1,628 @@ +From d57ff7a993fc74e4f10bc1cc1619d75ec93d2af1 Mon Sep 17 00:00:00 2001 +From: liyuanr +Date: Wed, 21 Aug 2024 16:52:16 +0800 +Subject: [PATCH 9/9] bugfix: fix the problem that proxy will get all os for + processing + +The OS modification triggers the reconcile of the proxy. As a result, the proxy processes +the OS upgrade or configuration that does not correspond to the current node. +To solve this problem, the namedspace field is added to osinstance.spec. During each upgrade +or configuration compelets, the name and namespace of the OS corresponding to osinstance are specified. +After the upgrade or configuration is complete, the field is set to None. + +Signed-off-by: liyuanr +--- + KubeOS-Rust/proxy/src/controller/apiclient.rs | 4 +- + .../proxy/src/controller/apiserver_mock.rs | 9 +++- + .../proxy/src/controller/controller.rs | 31 ++++++++----- + KubeOS-Rust/proxy/src/controller/crd.rs | 17 +++++++ + KubeOS-Rust/proxy/src/controller/values.rs | 3 +- + Makefile | 2 +- + api/v1alpha1/os_types.go | 11 +++++ + cmd/operator/controllers/operation.go | 14 +++--- + cmd/operator/controllers/os_controller.go | 44 ++++++------------- + cmd/operator/controllers/requirements.go | 5 ++- + cmd/operator/controllers/times.go | 24 ++++++---- + .../config/crd/upgrade.openeuler.org_os.yaml | 1 + + .../upgrade.openeuler.org_osinstances.yaml | 8 ++++ + docs/example/config/manager/manager.yaml | 7 +++ + pkg/values/values.go | 2 + + 15 files changed, 119 insertions(+), 63 deletions(-) + +diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs +index 3afd5a51..85e839f0 100644 +--- a/KubeOS-Rust/proxy/src/controller/apiclient.rs ++++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs +@@ -39,7 +39,7 @@ impl Default for OSInstanceSpecPatch { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), +- spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, ++ spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname:None}, + } + } + } +@@ -102,7 +102,7 @@ impl ApplyApi for ControllerClient { + labels: Some(labels), + ..ObjectMeta::default() + }, +- spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, ++ spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname: None}, + status: None, + }; + let osi_api = Api::namespaced(self.client.clone(), namespace); +diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs +index 2b182ca8..679096d3 100644 +--- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs ++++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs +@@ -33,7 +33,7 @@ use mockall::mock; + use self::mock_error::Error; + use super::{ + agentclient::*, +- crd::{Configs, OSInstanceStatus}, ++ crd::{Configs, NamespacedName, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK}, + }; + use crate::controller::{ +@@ -531,6 +531,7 @@ impl OSInstance { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), ++ namespacedname:Some(NamespacedName{namespace:String::from("default"),name:String::from("test")}) + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), +@@ -605,7 +606,7 @@ impl OS { + os.meta_mut().namespace = Some("default".into()); + os + } +- ++ + pub fn set_os_osversion_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); +@@ -676,6 +677,10 @@ impl Default for OSSpec { + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), ++ nodeselector:None, ++ timeinterval:None, ++ timewindow:None, ++ executionmode:None, + } + } + } +diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs +index baf2f9d8..40405b2d 100644 +--- a/KubeOS-Rust/proxy/src/controller/controller.rs ++++ b/KubeOS-Rust/proxy/src/controller/controller.rs +@@ -24,16 +24,14 @@ use kube::{ + use log::{debug, error, info}; + use reconciler_error::Error; + +-use crate::controller::values::LABEL_CONFIGURING; +- + use super::{ + agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + apiclient::ApplyApi, + crd::{Configs, Content, OSInstance, OS}, + utils::{check_version, get_config_version, ConfigOperation, ConfigType}, + values::{ +- LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, +- REQUEUE_ERROR, REQUEUE_NORMAL, ++ LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, ++ OSINSTANCE_NAMESPACE, REQUEUE_ERROR, REQUEUE_NORMAL,LABEL_CONFIGURING, NO_REQUEUE, + }, + }; + +@@ -48,10 +46,20 @@ pub async fn reconcile( + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; +- proxy_controller.check_osi_exisit(&namespace, &node_name).await?; +- let controller_res = proxy_controller.get_resources(&namespace, &node_name).await?; ++ proxy_controller.check_osi_exisit(&node_name).await?; ++ let controller_res = proxy_controller.get_resources(&node_name).await?; + let node = controller_res.node; + let mut osinstance = controller_res.osinstance; ++ if let Some(namespacedname) = osinstance.spec.namespacedname.as_ref(){ ++ debug!("osinstance correspending os name is {}, namespace is {}",namespacedname.name,namespacedname.namespace); ++ if !(namespacedname.name == os_cr.name() && namespacedname.namespace == namespace){ ++ debug!("current os cr name:{}, namespace:{} is not belong to this node",os_cr.name(),namespace); ++ return Ok(NO_REQUEUE) ++ } ++ }else { ++ return Ok(REQUEUE_NORMAL) ++ } ++ + let node_os_image = &node + .status + .as_ref() +@@ -153,8 +161,8 @@ impl ProxyController { + } + + impl ProxyController { +- async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { +- let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); ++ async fn check_osi_exisit(&self, node_name: &str) -> Result<(), Error> { ++ let osi_api: Api = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE); + match osi_api.get(node_name).await { + Ok(osi) => { + debug!("osinstance is exist {:?}", osi.name()); +@@ -162,15 +170,15 @@ impl ProxyController { + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + info!("Create OSInstance {}", node_name); +- self.controller_client.create_osinstance(node_name, namespace).await?; ++ self.controller_client.create_osinstance(node_name, OSINSTANCE_NAMESPACE).await?; + Ok(()) + }, + Err(err) => Err(Error::KubeClient { source: err }), + } + } + +- async fn get_resources(&self, namespace: &str, node_name: &str) -> Result { +- let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); ++ async fn get_resources(&self, node_name: &str) -> Result { ++ let osi_api: Api = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE); + let osinstance_cr = osi_api.get(node_name).await?; + let node_api: Api = Api::all(self.k8s_client.clone()); + let node_cr = node_api.get(node_name).await?; +@@ -234,6 +242,7 @@ impl ProxyController { + value: String::from("namespace"), + })?; + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); ++ osinstance.spec.namespacedname = None; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + } + Ok(()) +diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs +index 41f333e8..36e14d59 100644 +--- a/KubeOS-Rust/proxy/src/controller/crd.rs ++++ b/KubeOS-Rust/proxy/src/controller/crd.rs +@@ -32,6 +32,11 @@ pub struct OSSpec { + pub clientkey: Option, + pub sysconfigs: Option, + pub upgradeconfigs: Option, ++ pub nodeselector:Option, ++ pub timewindow: Option, ++ pub timeinterval: Option, ++ pub executionmode:Option, ++ + } + + #[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +@@ -46,6 +51,7 @@ pub struct OSSpec { + )] + pub struct OSInstanceSpec { + pub nodestatus: String, ++ pub namespacedname: Option, + pub sysconfigs: Option, + pub upgradeconfigs: Option, + } +@@ -75,3 +81,14 @@ pub struct Content { + pub value: Option, + pub operation: Option, + } ++ ++#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] ++pub struct NamespacedName{ ++ pub namespace: String, ++ pub name: String, ++} ++#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] ++pub struct TimeWindow{ ++ pub starttime:String, ++ pub endtime:String, ++} +\ No newline at end of file +diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs +index 9066f57d..2edb3dcb 100644 +--- a/KubeOS-Rust/proxy/src/controller/values.rs ++++ b/KubeOS-Rust/proxy/src/controller/values.rs +@@ -19,6 +19,7 @@ pub const LABEL_CONFIGURING: &str = "upgrade.openeuler.org/configuring"; + + pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; + pub const OSINSTANCE_KIND: &str = "OSInstance"; ++pub const OSINSTANCE_NAMESPACE: &str = "default"; + + pub const NODE_STATUS_IDLE: &str = "idle"; + pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +@@ -30,5 +31,5 @@ pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; + pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; + + pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; +- + pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; ++pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; +diff --git a/Makefile b/Makefile +index d4cd71e3..7a8db31d 100644 +--- a/Makefile ++++ b/Makefile +@@ -111,7 +111,7 @@ generate: controller-gen + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + + # Build the docker image +-docker-build: operator proxy ++docker-build: operator rust-proxy + docker build --target operator -t ${IMG_OPERATOR} . + docker build --target proxy -t ${IMG_PROXY} . + +diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go +index 6439bc14..bde86d25 100644 +--- a/api/v1alpha1/os_types.go ++++ b/api/v1alpha1/os_types.go +@@ -46,6 +46,7 @@ type OSSpec struct { + // +kubebuilder:validation:Optional + TimeWindow TimeWindow `json:"timewindow"` + // +kubebuilder:validation:Optional ++ // +kubebuilder:default:=15 + TimeInterval int `json:"timeinterval"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=serial;parallel +@@ -132,11 +133,21 @@ type OSInstanceSpec struct { + // +kubebuilder:validation:Optional + NodeStatus string `json:"nodestatus"` + // +kubebuilder:validation:Optional ++ NamespacedName NamespacedName `json:"namespacedname"` ++ // +kubebuilder:validation:Optional + SysConfigs SysConfigs `json:"sysconfigs"` + // +kubebuilder:validation:Optional + UpgradeConfigs SysConfigs `json:"upgradeconfigs"` + } + ++// NamespacedName defines name and namespace of os corresponding to osinstance ++type NamespacedName struct { ++ // +kubebuilder:validation:Optional ++ Name string `json:"name"` ++ // +kubebuilder:validation:Optional ++ Namespace string `json:"namespace"` ++} ++ + // +kubebuilder:object:root=true + + // OSInstanceList is a list of OSInstance +diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go +index dd0b4fec..5ac3d6d4 100644 +--- a/cmd/operator/controllers/operation.go ++++ b/cmd/operator/controllers/operation.go +@@ -54,7 +54,7 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, + if os.Spec.OSVersion != osVersionNode { + log.Info("Upgrading node " + node.Name) + var osInstance upgradev1.OSInstance +- if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { ++ if err := r.Get(ctx, types.NamespacedName{Namespace: values.OsiNamespace, Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") +@@ -73,7 +73,7 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, + } + } + if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { +- if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { ++ if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 { + log.Error(nil, "failed to delete nodes serial label") + } + } +@@ -123,11 +123,14 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW + } + } + osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() ++ osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace} ++ log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name) + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + log.Info("Update osinstance spec successfully") ++ + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) +@@ -175,7 +178,7 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o + } + } + if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { +- if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { ++ if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 { + log.Error(nil, "failed to delete nodes serial label") + } + } +@@ -192,11 +195,14 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr + return err + } + osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() ++ osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace} ++ log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name) + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + log.Info("Update osinstance spec successfully") ++ + node.Labels[values.LabelConfiguring] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) +@@ -219,7 +225,6 @@ func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o + var count int + var errList []error + for _, node := range nodes { +- log.V(1).Info("start add serial label to nodes") + if count >= limit { + break + } +@@ -267,7 +272,6 @@ func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o + } + func (s serialOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { +- log.V(1).Info("start update nodes") + node.Labels[values.LabelSerial] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) +diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go +index 667a265f..9e2e8e49 100644 +--- a/cmd/operator/controllers/os_controller.go ++++ b/cmd/operator/controllers/os_controller.go +@@ -66,13 +66,13 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) + } + return values.RequeueNow, err + } +- log.V(1).Info("get os cr name is " + os.Name + ", namespace is " + os.Namespace) + isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) + if err != nil { + return values.RequeueNow, err + } + if !isWithinTimeWindow { +- log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + " , the start time " + os.Spec.TimeWindow.StartTime) ++ log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + ++ " , the end time " + os.Spec.TimeWindow.EndTime) + return values.Requeue, nil + } + +@@ -106,7 +106,6 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) + if err != nil { + return values.RequeueNow, err + } +- log.V(1).Info("get all nodes is " + strconv.Itoa(len(allNodes))) + switch os.Spec.ExecutionMode { + case ExecutionModeParallel: + result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes)) +@@ -139,28 +138,6 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { + }); err != nil { + return err + } +- // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", +- // func(rawObj client.Object) []string { +- // os, ok := rawObj.(*upgradev1.OS) +- // if !ok { +- // log.Error(nil, "failed to convert to osInstance") +- // return []string{} +- // } +- // return []string{os.Name} +- // }); err != nil { +- // return err +- // } +- // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", +- // func(rawObj client.Object) []string { +- // os, ok := rawObj.(*upgradev1.OS) +- // if !ok { +- // log.Error(nil, "failed to convert to osInstance") +- // return []string{} +- // } +- // return []string{os.Namespace} +- // }); err != nil { +- // return err +- // } + return ctrl.NewControllerManagedBy(mgr). + For(&upgradev1.OS{}). + Watches(&source.Kind{Type: &corev1.Node{}}, handler.Funcs{DeleteFunc: r.DeleteOSInstance}). +@@ -219,12 +196,14 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + } + func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + opsInstance operation, requirements []labels.Requirement) (int, error) { ++ if limit == 0 { ++ return 0, nil ++ } + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated + if err != nil { + return 0, err + } +- log.V(1).Info("get need nodes is " + strconv.Itoa(len(nodes))) +- // Upgrade OS for selected nodes ++ log.V(1).Info("get wait to check nodes is " + strconv.Itoa(len(nodes))) + count, errLists := opsInstance.updateNodes(ctx, r, &os, nodes, limit) + if len(errLists) != 0 { + return 0, fmt.Errorf("update nodes and osinstance error") +@@ -343,7 +322,7 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up + if len(opsNodeNum) > 0 { + return values.Requeue, nil + } +- log.V(1).Info("get opsnodes is " + strconv.Itoa(len(opsNodeNum))) ++ + serialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, + selection.Equals, selection.Exists).createNodeRequirement(ctx, r) + if err != nil { +@@ -353,15 +332,20 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up + if err != nil { + return values.RequeueNow, nil + } +- log.V(1).Info("get serialLimit is " + strconv.Itoa(serialNodeLimit)) ++ + noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, + selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } +- if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOps{opsInsatnce.getOpsLabel()}, noSerialNodesRequirement); err != nil { ++ // add serial label to node ++ serialOpsInstance := serialOps{ ++ label: opsInsatnce.getOpsLabel(), ++ } ++ if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOpsInstance, noSerialNodesRequirement); err != nil { + return values.RequeueNow, nil + } ++ + serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial + count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement) + if err != nil { +diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go +index 9317b495..12db8e82 100644 +--- a/cmd/operator/controllers/requirements.go ++++ b/cmd/operator/controllers/requirements.go +@@ -18,6 +18,7 @@ import ( + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" ++ + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" + ) +@@ -124,8 +125,8 @@ type commonsNodesRequirement struct { + + func newCommonsNodesRequirement(nodeSelector string, op selection.Operator) commonsNodesRequirement { + return commonsNodesRequirement{ +- nodeSelector, +- op, ++ nodeSelector: nodeSelector, ++ op: op, + } + } + +diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go +index eab3a224..3a72cce9 100644 +--- a/cmd/operator/controllers/times.go ++++ b/cmd/operator/controllers/times.go +@@ -25,6 +25,7 @@ const ( + TIME_ONLY = "15:04:05" + ExecutionModeSerial = "serial" + ExecutionModeParallel = "parallel" ++ oneDayTime = 24 * time.Hour + ) + + func isWithinTimeWindow(start, end string) (bool, error) { +@@ -32,7 +33,7 @@ func isWithinTimeWindow(start, end string) (bool, error) { + return true, nil + } + if start == "" || end == "" { +- return false, fmt.Errorf("The start time and end time must be both empty or not empty.") ++ return false, fmt.Errorf("invalid TimeWindow: The start time and end time must be both empty or not empty") + } + layoutStart, err := checkTimeValid(start) + if err != nil { +@@ -43,13 +44,11 @@ func isWithinTimeWindow(start, end string) (bool, error) { + return false, err + } + if layoutStart != layoutEnd { +- return false, fmt.Errorf("Start Time needs same time format with End Time") ++ return false, fmt.Errorf("invalid TimeWindow: Start Time should have same time format with End Time") + } + now := time.Now() +- if layoutStart == TIME_ONLY { +- timeFormat := now.Format(TIME_ONLY) +- now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) +- } ++ timeFormat := now.Format(layoutStart) ++ now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) + startTime, err := time.ParseInLocation(layoutStart, start, now.Location()) + if err != nil { + return false, err +@@ -58,11 +57,18 @@ func isWithinTimeWindow(start, end string) (bool, error) { + if err != nil { + return false, err + } ++ if endTime.Equal(startTime) { ++ return false, fmt.Errorf("invalid TimeWindow: start time is equal to end time") ++ } + if endTime.Before(startTime) { +- endTime = endTime.Add(24 * time.Hour) ++ if layoutStart == DATE_TIME { ++ return false, fmt.Errorf("invalid TimeWindow: Start %s Time is after end time %s", ++ startTime.Format(layoutStart), endTime.Format(layoutEnd)) ++ } ++ endTime = endTime.Add(oneDayTime) + fmt.Printf("endtime time add 24 hour is %s\n", endTime.Format(layoutStart)) + if now.Before(startTime) { +- now = now.Add(24 * time.Hour) ++ now = now.Add(oneDayTime) + fmt.Printf("now time add 24 hour is %s\n", now.Format(layoutStart)) + } + +@@ -96,5 +102,5 @@ func checkTimeValid(checkTime string) (string, error) { + return TIME_ONLY, nil + + } +- return "", fmt.Errorf("Invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS such as \"2006-01-02 15:04:05\" or \"15:04:05\"") ++ return "", fmt.Errorf("invalid TimeWindow: invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS") + } +diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml +index f64bad64..8a29e0c3 100644 +--- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml ++++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml +@@ -117,6 +117,7 @@ spec: + type: string + type: object + timeinterval: ++ default: 15 + type: integer + timewindow: + properties: +diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +index df9119b4..47de87e3 100644 +--- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml ++++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +@@ -53,6 +53,14 @@ spec: + spec: + description: OSInstanceSpec defines desired state of OS + properties: ++ namespacedname: ++ description: NamespacedName defines name and namespace of os corresponding to osinstance ++ properties: ++ name: ++ type: string ++ namespace: ++ type: string ++ type: object + nodestatus: + type: string + sysconfigs: +diff --git a/docs/example/config/manager/manager.yaml b/docs/example/config/manager/manager.yaml +index f5cf84f9..c35d9010 100644 +--- a/docs/example/config/manager/manager.yaml ++++ b/docs/example/config/manager/manager.yaml +@@ -60,6 +60,9 @@ spec: + - /operator + image: + name: operator ++ volumeMounts: ++ - name: date-config ++ mountPath: /etc/localtime + securityContext: + allowPrivilegeEscalation: false + runAsUser: 6552 +@@ -71,6 +74,10 @@ spec: + requests: + cpu: 100m + memory: 20Mi ++ volumes: ++ - name: date-config ++ hostPath: ++ path: /etc/localtime + terminationGracePeriodSeconds: 10 + nodeSelector: + node-role.kubernetes.io/control-plane: "" +diff --git a/pkg/values/values.go b/pkg/values/values.go +index 2045bbe6..9e213a83 100644 +--- a/pkg/values/values.go ++++ b/pkg/values/values.go +@@ -40,6 +40,8 @@ const ( + UpgradeConfigName = "UpgradeConfig" + // SysConfigName is param name of SysConfig + SysConfigName = "SysConfig" ++ // OsiNamespace is the namespace of osinstance ++ OsiNamespace = "default" + ) + + // NodeStatus defines state of nodes +-- +2.33.0.windows.2 + diff --git a/KubeOS.spec b/KubeOS.spec index da3ec84..7dfcb47 100644 --- a/KubeOS.spec +++ b/KubeOS.spec @@ -2,12 +2,19 @@ Name: KubeOS Version: 1.0.6 -Release: 2 +Release: 3 Summary: O&M platform used to update the whole OS as an entirety License: Mulan PSL v2 Source0: https://gitee.com/openeuler/KubeOS/repository/archive/v%{version}.tar.gz Patch1: 0001-Bump-kubeos-version-to-1.0.6.patch Patch2: 0002-fix-update-bootloader.sh-paths-for-EFI-boot.patch +Patch3: 0003-docs-update-readme-and-format-yaml.patch +Patch4: 0004-KubeOS-add-nodeselector-to-OS.patch +Patch5: 0005-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch +Patch6: 0006-operator-delete-unnecessary-fmt-and-add-printing-for.patch +Patch7: 0007-feat-os-operator-support-setting-TimeWindow-and-Time.patch +Patch8: 0008-feat-os-proxy-add-ExcutionMode-to-os.patch +Patch9: 0009-bugfix-fix-the-problem-that-proxy-will-get-all-os-fo.patch BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: make rust cargo openssl-devel @@ -119,6 +126,12 @@ install -p -m 0600 ./files/os-release %{buildroot}/opt/kubeOS/files rm -rfv %{buildroot} %changelog +* Tue Jun 11 2024 Yuhang Wei - 1.0.6-3 +- Type:requirement +- CVE:NA +- SUG:restart +- DESC:sync from code from upstream master branch + * Tue Jun 11 2024 Yuhang Wei - 1.0.6-2 - Type:requirement - CVE:NA -- Gitee