Kubernetes 最佳实践

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum install -y https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
yum -y --disablerepo="*" --enablerepo="elrepo-kernel" list available
yum -y --disablerepo=\* --enablerepo=elrepo-kernel install kernel-lt.x86_64
awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg

grub2-set-default 0
# 这里的 0 要根据实际情况来填写

reboot

[root@lab-kubernetes ~]# uname -a
Linux lab-kubernetes 5.4.196-1.el7.elrepo.x86_64 #1 SMP Tue May 24 12:49:20 EDT 2022 x86_64 x86_64 x86_64 GNU/Linux

yum remove python3

yum update -y
yum install centos-release-scl-rh -y
yum install rh-python38-python -y
yum install rh-python38-python-pip -y
yum install rh-python38-python-devel -y

echo "export PATH=$PATH:/opt/rh/rh-python38/root/usr/bin" >> /etc/profile && source /etc/profile

mkdir ~/.pip
vi ~/.pip/pip.conf

[global]
index-url=https://pypi.tuna.tsinghua.edu.cn/simple/

python3 -m pip install --upgrade pip

[root@lab-c2009 ~]# python3 --version
Python 3.8.11

[root@lab-c2009 ~]# python3 -m pip --version
pip 22.0.4 from /opt/rh/rh-python38/root/usr/local/lib/python3.8/site-packages/pip (python 3.8)

yum install curl-devel expat-devel gettext-devel openssl-devel zlib-devel asciidoc -y
yum install gcc perl-ExtUtils-MakeMaker -y

yum remove git -y

cd /usr/local/src/
wget https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.32.0.tar.xz
tar -xvf git-2.32.0.tar.xz
cd git-2.32.0

make prefix=/usr/local/git all
make prefix=/usr/local/git install

echo "export PATH=$PATH:/usr/local/git/bin" >> /etc/profile && source /etc/profile

python3 -m pip install git-review
echo "export PATH=$PATH:/opt/rh/rh-python38/root/usr/local/bin" >> /etc/profile && source /etc/profile

[root@lab-kubernetes git-2.32.0]# git --version
git version 2.32.0

[root@lab-kubernetes ~]# docker run -d nginx
614b35ef3e2842cdc57bcb08c89df93533802d443065f8887c3951a601b738c2

[root@lab-kubernetes ~]# docker run -d nginx
62970cd2a51798b07c4f8ad6b42a19e72410aba7a2e6841c1c1606a35feedbee

[root@lab-kubernetes ~]# ps -ef
root     18442     1  0 00:44 ?        00:00:27 /usr/bin/containerd
root     18452     1  0 00:44 ?        00:00:02 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
root     20595     1  0 11:18 ?        00:00:00 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 614b35ef3e2842cdc57bcb08c89df93533802d443065f8887c3951a601b738c2 -address /run/containerd/containerd.sock
root     20615 20595  0 11:18 ?        00:00:00 nginx: master process nginx -g daemon off;
root     20708     1  0 11:22 ?        00:00:00 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 62970cd2a51798b07c4f8ad6b42a19e72410aba7a2e6841c1c1606a35feedbee -address /run/containerd/containerd.sock
root     20728 20708  0 11:22 ?        00:00:00 nginx: master process nginx -g daemon off;

# 更新依赖仓库
apt-get update -y || yum update -y

# 安装 Docker
apt-get install docker.io -y || yum install docker -y
systemctl enable docker --now

# 检查 docker 服务状态
systemctl status docker

# ubuntu 中需要把 docker 的 cgroup driver 改成 systemd
# !! centos 默认就是 systemd，不要修改这个文件，改了 docker 会起不来，保持 {} 就好
vi /etc/docker/daemon.json

{
  "exec-opts": ["native.cgroupdriver=systemd"]
}

systemctl restart docker

# run hello-world
docker run hello-world

mkdir ~/test
cd ~/test

wget https://gitee.com/dev-99cloud/lab-openstack/raw/master/src/docker-quickstart/app.py
wget https://gitee.com/dev-99cloud/lab-openstack/raw/master/src/docker-quickstart/requirements.txt
wget https://gitee.com/dev-99cloud/lab-openstack/raw/master/src/docker-quickstart/Dockerfile

# 如果是 CentOS 7.x 需要安装下 python3
# yum install python3 python3-pip

# python3 -m pip install -r requirements.txt
python3 -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt

python3 app.py
# * Running on http://0.0.0.0:80/ (Press CTRL+C to quit)
# 此时可以从浏览器访问 http://<ip>/

docker build --tag=friendlyhello .

# 可以看一下本地镜像列表
docker images

docker rm testFlask
docker run --rm -p 4000:80 --name=testFlask friendlyhello

# 也可以从 dockerhub 下载
docker run --rm -p 4000:80 --name=testFlask 99cloud/friendlyhello:3.9.6
# * Running on http://0.0.0.0:80/ (Press CTRL+C to quit)
# 此时可以从浏览器访问 http://<ip>:4000

# 如果要跑在后台，可以加 -d 参数
docker run -d --rm -p 4000:80 --name=testNew 99cloud/friendlyhello:3.9.6

# 进入容器调试
$ docker exec -it testFlask /bin/bash
root@4224b69e7ee3:/app# env
HOSTNAME=4224b69e7ee3
PWD=/app
HOME=/root
NAME=World
...

root@4224b69e7ee3:/app# ps -ef

# 查看容器日志
docker logs -f testFlask

# 结束容器
docker stop testFlask

# 启动容器
docker start testFlask

# 从容器生成新的镜像
docker stop testFlask
docker commit testFlask test-new

# 删除容器
docker rm testFlask

# 删除镜像
docker rmi maodouzi/get-started:part2

[root@cloud025 ns]# ll /proc/2179/ns/
total 0
lrwxrwxrwx 1 root root 0 Aug 10 11:58 ipc -> ipc:[4026531839]
lrwxrwxrwx 1 root root 0 Aug 10 11:58 mnt -> mnt:[4026531840]
lrwxrwxrwx 1 root root 0 Aug 10 11:58 net -> net:[4026531956]
lrwxrwxrwx 1 root root 0 Aug 10 11:58 pid -> pid:[4026531836]
lrwxrwxrwx 1 root root 0 Aug 10 11:58 user -> user:[4026531837]
lrwxrwxrwx 1 root root 0 Aug 10 11:58 uts -> uts:[4026531838]

[root@cloud025 ns]# docker ps
CONTAINER ID        IMAGE                         COMMAND                  CREATED             STATUS              PORTS                  NAMES
07297a3ac7ea        nginx:latest                  "/docker-entrypoin..."   29 minutes ago      Up 29 minutes       80/tcp                 devtest
0935b08509a4        test-new                      "python app.py"          35 minutes ago      Up 35 minutes       0.0.0.0:5000->80/tcp   testNew
c23c76dd779c        99cloud/friendlyhello:3.9.6   "python app.py"          37 minutes ago      Up 36 minutes       0.0.0.0:4000->80/tcp   testFlask

[root@cloud025 ns]# docker inspect testFlask | grep -i pid
            "Pid": 1159,
            "PidMode": "",
            "PidsLimit": 0,

[root@cloud025 ns]# mkdir -p /var/run/netns

[root@cloud025 ns]# ln -s /proc/1159/ns/net /var/run/netns/testFlask

[root@cloud025 ns]# ip netns list
testFlask (id: 0)
devtest (id: 2)
testNew (id: 1)

[root@cloud025 ns]# ip netns exec testNew ip a
...
44: eth0@if45: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:ac:11:00:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 172.17.0.3/16 scope global eth0
      valid_lft forever preferred_lft forever
    inet6 fe80::42:acff:fe11:3/64 scope link
      valid_lft forever preferred_lft forever

[root@cloud025 ns]# ip a
45: vethb6d08be@if44: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master docker0 state UP group default
    link/ether 0e:d9:14:d1:86:98 brd ff:ff:ff:ff:ff:ff link-netnsid 1
    inet6 fe80::cd9:14ff:fed1:8698/64 scope link
      valid_lft forever preferred_lft forever

[root@cloud025 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
docker0		8000.02428c25c112	no		vethb6d08be
virbr0		8000.525400d583b9	yes		virbr0-nic

# 查看指定 pod 运行在那个 node 上
kubectl describe pod <pod> -n <namespace>

# 获得容器的 pid
docker inspect -f {{.State.Pid}} <container>

# 进入该容器的 network namespace
nsenter --target <PID> -n

# 使用 tcpdump 抓包，指定 eth0 网卡
tcpdump -i eth0 tcp and port 80 -vvv

# 或者抓包并导出到文件
tcpdump -i eth0 -w ./out.cap

# 退出 nsenter，要记得退出！！
exit

VERSION="v1.24.1"
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-amd64.tar.gz
tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
rm -f crictl-$VERSION-linux-amd64.tar.gz

[root@lab-kubernetes tmp]# crictl --version
crictl version v1.24.1

[root@lab-kubernetes tmp]# crictl images
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]. As the default settings are now deprecated, you should set the endpoint instead.
ERRO[0002] connect endpoint 'unix:///var/run/dockershim.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
ERRO[0004] connect endpoint 'unix:///run/containerd/containerd.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
FATA[0006] connect: connect endpoint 'unix:///run/crio/crio.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded

yum remove -y docker \
            docker-client \
            docker-client-latest \
            docker-common \
            docker-latest \
            docker-latest-logrotate \
            docker-logrotate \
            docker-engine

yum install -y yum-utils
yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo

yum install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin

# yum list docker-ce --showduplicates | sort -r
# yum install docker-ce-20.10.16 docker-ce-cli-20.10.16 containerd.io docker-compose-plugin

systemctl enable docker --now

echo "alias docker-compose='docker compose'" >> /etc/profile && source /etc/profile

docker version
docker-compose version

[root@lab-kubernetes ~]# cat /usr/lib/systemd/system/docker.service

[Unit]
...
Requires=docker.socket containerd.service

[Service]
...
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

[root@lab-kubernetes ~]# cat /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock

[root@lab-kubernetes ~]# crictl images
FATA[0000] listing images: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.ImageService

[root@lab-kubernetes ~]# vi /etc/containerd/config.toml

[root@lab-kubernetes ~]# cat /etc/containerd/config.toml | grep cri
# disabled_plugins = ["cri"]

[root@lab-kubernetes ~]# systemctl restart containerd

[root@lab-kubernetes ~]# crictl images
IMAGE               TAG                 IMAGE ID            SIZE

[root@lab-kubernetes ~]# crictl ps -a
CONTAINER           IMAGE               CREATED             STATE               NAME                ATTEMPT             POD ID

ctr version

# pull 镜像
# 注意 ⚠️，这里要按 registry/orgnization/image:tag 写全，没有默认了
ctr images pull docker.io/library/redis:alpine3.13

# 查看镜像
ctr i ls

# 启动镜像
ctr run -d docker.io/library/redis:alpine3.13 redis

ctr container ls
CONTAINER    IMAGE                                 RUNTIME
redis        docker.io/library/redis:alpine3.13    io.containerd.runc.v2

ctr task ls
TASK     PID      STATUS
redis    20808    RUNNING

ctr task exec -t --exec-id redis-sh redis sh
/data # redis-cli
127.0.0.1:6379> set k1 v1
OK
127.0.0.1:6379> get k1
"v1"

$ ps -ef | grep runc | grep redis
/usr/local/containerd/bin/containerd-shim-runc-v2 -namespace default -id redis -address /run/containerd/containerd.sock

$ ctr ns ls
NAME    LABELS
default
k8s.io

ctr -n default i ls
ctr -n k8s.io i ls

$ crictl pull docker.io/library/redis:alpine3.13

$ crictl images
IMAGE                                       TAG                 IMAGE ID            SIZE
docker.io/library/redis                     alpine3.13          554d20f203657       10.9MB

[root@lab-kubernetes ~]# ctr plugin ls
TYPE                                  ID                       PLATFORMS      STATUS
io.containerd.content.v1              content                  -              ok
io.containerd.snapshotter.v1          aufs                     linux/amd64    skip
io.containerd.snapshotter.v1          btrfs                    linux/amd64    skip
io.containerd.snapshotter.v1          devmapper                linux/amd64    error
io.containerd.snapshotter.v1          native                   linux/amd64    ok
io.containerd.snapshotter.v1          overlayfs                linux/amd64    ok
io.containerd.snapshotter.v1          zfs                      linux/amd64    skip
io.containerd.metadata.v1             bolt                     -              ok
io.containerd.differ.v1               walking                  linux/amd64    ok
io.containerd.event.v1                exchange                 -              ok
io.containerd.gc.v1                   scheduler                -              ok
io.containerd.service.v1              introspection-service    -              ok
io.containerd.service.v1              containers-service       -              ok
io.containerd.service.v1              content-service          -              ok
io.containerd.service.v1              diff-service             -              ok
io.containerd.service.v1              images-service           -              ok
io.containerd.service.v1              leases-service           -              ok
io.containerd.service.v1              namespaces-service       -              ok
io.containerd.service.v1              snapshots-service        -              ok
io.containerd.runtime.v1              linux                    linux/amd64    ok
io.containerd.runtime.v2              task                     linux/amd64    ok
io.containerd.monitor.v1              cgroups                  linux/amd64    ok
io.containerd.service.v1              tasks-service            -              ok
io.containerd.grpc.v1                 introspection            -              ok
io.containerd.internal.v1             restart                  -              ok
io.containerd.grpc.v1                 containers               -              ok
io.containerd.grpc.v1                 content                  -              ok
io.containerd.grpc.v1                 diff                     -              ok
io.containerd.grpc.v1                 events                   -              ok
io.containerd.grpc.v1                 healthcheck              -              ok
io.containerd.grpc.v1                 images                   -              ok
io.containerd.grpc.v1                 leases                   -              ok
io.containerd.grpc.v1                 namespaces               -              ok
io.containerd.internal.v1             opt                      -              ok
io.containerd.grpc.v1                 snapshots                -              ok
io.containerd.grpc.v1                 tasks                    -              ok
io.containerd.grpc.v1                 version                  -              ok
io.containerd.tracing.processor.v1    otlp                     -              skip
io.containerd.internal.v1             tracing                  -              ok
io.containerd.grpc.v1                 cri                      linux/amd64    ok

...
[plugins]
...
  [plugins."io.containerd.grpc.v1.cri"]
  ...
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.my.org"]
          endpoint = ["https://harbor.my.org"]
      [plugins."io.containerd.grpc.v1.cri".registry.configs]
        [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.my.org".tls]
          insecure_skip_verify = true
        [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.my.org".auth]
          username = "username"
          password = "passwd"
          # auth = "base64(username:password)"
...

crictl pull harbor.my.org/library/nginx:1.1

wget https://github.com/containerd/nerdctl/releases/download/v0.20.0/nerdctl-0.20.0-linux-amd64.tar.gz

tar zxvf nerdctl-0.20.0-linux-amd64.tar.gz -C /usr/bin/
nerdctl run -d --name nginx -p 80:80 nginx:alpine
ctr -n default i ls

yum install -y podman

podman pull docker.io/library/httpd
podman images

podman run -dt -p 8080:80/tcp docker.io/library/httpd

[root@lab-kubernetes ~]# podman run -dt -p 8080:80/tcp docker.io/library/httpd
Trying to pull docker.io/library/httpd...
439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04

[root@lab-kubernetes ~]# ps -ef | grep httpd
root     22267 22256  0 17:01 pts/0    00:00:00 httpd -DFOREGROUND
33       22278 22267  0 17:01 pts/0    00:00:00 httpd -DFOREGROUND
33       22279 22267  0 17:01 pts/0    00:00:00 httpd -DFOREGROUND
33       22280 22267  0 17:01 pts/0    00:00:00 httpd -DFOREGROUND
root     22256     1  0 17:01 ?        00:00:00 /usr/bin/conmon --api-version 1 -s -c 439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04 -u 439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04 -r /usr/bin/runc -b /var/lib/containers/storage/overlay-containers/439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04/userdata -p /var/run/containers/storage/overlay-containers/439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04/userdata/pidfile -l k8s-file:/var/lib/containers/storage/overlay-containers/439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04/userdata/ctr.log --exit-dir /var/run/libpod/exits --socket-dir-path /var/run/libpod/socket --log-level error --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/var/run/containers/storage/overlay-containers/439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04/userdata/oci-log -t --conmon-pidfile /var/run/containers/storage/overlay-containers/439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /var/run/containers/storage --exit-command-arg --log-level --exit-command-arg error --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /var/run/libpod --exit-command-arg --runtime --exit-command-arg runc --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg 439e9d1532d0578af0d3d80e9b06f728ed06389a6d878430b8bee71aa74f5d04

yum remove runc -y

# 配置 containerd 所需 kernel module
cat << EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

# 加载 kernel module
modprobe overlay
modprobe br_netfilter

# 修改内核参数
cat << EOF > /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
user.max_user_namespaces=28633
EOF
sysctl -p /etc/sysctl.d/99-kubernetes-cri.conf

# 低内核版本 nf_conntrack_ipv4 / 高内核版本（4.x+） nf_conntrack
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack

yum install -y ipset ipvsadm

# 下载和安装 containerd
wget https://github.com/containerd/containerd/releases/download/v1.6.5/cri-containerd-cni-1.6.5-linux-amd64.tar.gz
tar -zxvf cri-containerd-cni-1.6.5-linux-amd64.tar.gz -C /

[root@lab-kubernetes tmp]# tar -zxvf cri-containerd-cni-1.6.5-linux-amd64.tar.gz -C /
etc/
etc/cni/
etc/cni/net.d/
etc/cni/net.d/10-containerd-net.conflist
etc/crictl.yaml
etc/systemd/
etc/systemd/system/
etc/systemd/system/containerd.service
usr/
usr/local/
usr/local/sbin/
usr/local/sbin/runc
usr/local/bin/
usr/local/bin/containerd-shim
usr/local/bin/containerd
usr/local/bin/critest
usr/local/bin/crictl
usr/local/bin/containerd-shim-runc-v1
usr/local/bin/containerd-stress
usr/local/bin/containerd-shim-runc-v2
usr/local/bin/ctd-decoder
usr/local/bin/ctr
opt/
opt/cni/
opt/cni/bin/
opt/cni/bin/static
opt/cni/bin/vlan
opt/cni/bin/bandwidth
opt/cni/bin/portmap
opt/cni/bin/ipvlan
opt/cni/bin/firewall
opt/cni/bin/macvlan
opt/cni/bin/dhcp
opt/cni/bin/ptp
opt/cni/bin/sbr
opt/cni/bin/vrf
opt/cni/bin/loopback
opt/cni/bin/host-device
opt/cni/bin/tuning
opt/cni/bin/bridge
opt/cni/bin/host-local
opt/containerd/
opt/containerd/cluster/
opt/containerd/cluster/gce/
opt/containerd/cluster/gce/cloud-init/
opt/containerd/cluster/gce/cloud-init/node.yaml
opt/containerd/cluster/gce/cloud-init/master.yaml
opt/containerd/cluster/gce/configure.sh
opt/containerd/cluster/gce/cni.template
opt/containerd/cluster/gce/env
opt/containerd/cluster/version

wget https://github.com/opencontainers/runc/releases/download/v1.1.0-rc.1/runc.amd64
chmod +x runc.amd64
cp runc.amd64 /usr/local/sbin/runc

# 生成 containerd 配置文件
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  ...
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true
   ...
  [plugins."io.containerd.grpc.v1.cri"]
    sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.6"
   ...

# 然后启动 containerd
systemctl enable containerd --now

# 检查 containerd 是否运行正常
ctr version
ctr plugin ls
crictl version

wget https://download.docker.com/linux/static/stable/x86_64/docker-20.10.9.tgz

tar -xvf docker-20.10.9.tgz

[root@lab-kubernetes tmp]# tar -xvf docker-20.10.9.tgz
docker/
docker/containerd-shim-runc-v2
docker/dockerd
docker/docker-proxy
docker/ctr
docker/docker
docker/runc
docker/containerd-shim
docker/docker-init
docker/containerd

mv docker/docker* /usr/bin

cat <<EOF > /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=containerd.service

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd --containerd /run/containerd/containerd.sock --cri-containerd
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity

# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes

# kill only the docker process, not all processes in the cgroup
KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl start docker

[root@lab-kubernetes tmp]# docker run hello-world

Hello from Docker!
This message shows that your installation appears to be working correctly.

To generate this message, Docker took the following steps:
 1. The Docker client contacted the Docker daemon.
 2. The Docker daemon pulled the "hello-world" image from the Docker Hub.
    (amd64)
 3. The Docker daemon created a new container from that image which runs the
    executable that produces the output you are currently reading.
 4. The Docker daemon streamed that output to the Docker client, which sent it
    to your terminal.

To try something more ambitious, you can run an Ubuntu container with:
 $ docker run -it ubuntu bash

Share images, automate workflows, and more with a free Docker ID:
 https://hub.docker.com/

For more examples and ideas, visit:
 https://docs.docker.com/get-started/

[root@lab-kubernetes tmp]# docker version
Client:
 Version:           20.10.9
 API version:       1.41
 Go version:        go1.16.8
 Git commit:        c2ea9bc
 Built:             Mon Oct  4 16:03:22 2021
 OS/Arch:           linux/amd64
 Context:           default
 Experimental:      true

Server: Docker Engine - Community
 Engine:
  Version:          20.10.9
  API version:      1.41 (minimum version 1.12)
  Go version:       go1.16.8
  Git commit:       79ea9d3
  Built:            Mon Oct  4 16:07:30 2021
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          v1.6.5
  GitCommit:        96df0994faabc1944fc614e52b0b3c6feb609a57
 runc:
  Version:          1.1.0-rc.1
  GitCommit:        v1.1.0-rc.1-0-g55df1fc4
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

[root@kubernetes004 ~]# nerdctl run -d --name nginx -p 80:80 nginx:alpine
FATA[0000] failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: time="2022-06-07T21:21:59+08:00" level=fatal msg="failed to call cni.Setup: plugin type=\"bridge\" failed (add): incompatible CNI versions; config is \"1.0.0\", plugin supports [\"0.1.0\" \"0.2.0\" \"0.3.0\" \"0.3.1\" \"0.4.0\"]"
Failed to write to log, write /var/lib/nerdctl/1935db59/containers/default/954e3d09f83bdce90305b2e8f58dad59a58ac5d9f350711a4025c7854067470c/oci-hook.createRuntime.log: file already closed: unknown

[root@kubernetes004 ~]# nerdctl -h | grep cni
      --cni-netconfpath string   cni config directory [$NETCONFPATH] (default "/etc/cni/net.d")
      --cni-path string          cni plugins binary directory [$CNI_PATH] (default "/usr/libexec/cni")

[root@kubernetes004 ~]# ls /usr/libexec/cni/
bandwidth    dhcp         flannel      host-local   loopback     portmap      sample       static       vlan
bridge       firewall     host-device  ipvlan       macvlan      ptp          sbr          tuning

[root@kubernetes004 ~]# rm -rf /usr/libexec/cni/

[root@kubernetes004 ~]# nerdctl -h | grep cni
      --cni-netconfpath string   cni config directory [$NETCONFPATH] (default "/etc/cni/net.d")
      --cni-path string          cni plugins binary directory [$CNI_PATH] (default "/opt/cni/bin")

[root@kubernetes004 ~]# nerdctl rm -f nginx
[root@kubernetes004 ~]# nerdctl run -d --name nginx -p 80:80 nginx:alpine
4958c5b1ad1fd2db2d8d1c4ccd0d796cd9115d4f07f17aa7a1fa615b89831c7a

OS=CentOS_7
VERSION=1.23
curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/devel:kubic:libcontainers:stable.repo
curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.repo https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/devel:kubic:libcontainers:stable:cri-o:$VERSION.repo
yum install -y cri-o

vim /lib/systemd/system/kubelet.service

[Unit]
......
[Service]
......
ExecStart=/k8s/kubernetes/bin/kubelet $KUBELET_OPTS --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --cgroup-driver=systemd
......

[Install]
.......

systemctl start crio

# 使用 crictl 查看版本
crictl version

# 重启 kubelet
systemctl restart kubelet

$ crictl -r /var/run/crio/crio.sock version
Version:  0.1.0
RuntimeName:  cri-o
RuntimeVersion:  1.11.11-1.rhaos3.11.git474f73d.el7
RuntimeApiVersion:  v1alpha1

$ crictl -r /var/run/crio/crio.sock ps
CONTAINER           IMAGE                                                                                               CREATED             STATE               NAME                ATTEMPT             POD ID
fbe5b37ad3c47       docker.io/library/busybox@sha256:895ab622e92e18d6b461d671081757af7dbaa3b00e3e28e12505af7817f73649   About an hour ago   Running             busybox             0                   f9f42b5ae54c7

$ docker ps
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES

[root@node02 runc]# runc list
ID                                                                 PID         STATUS      BUNDLE                                                                                                                 CREATED                          OWNER
f9f42b5ae54c71180ab7eb1706205bba79597d019f1c3e22f5f68e0b0ae055a8   31252       running     /run/containers/storage/overlay-containers/f9f42b5ae54c71180ab7eb1706205bba79597d019f1c3e22f5f68e0b0ae055a8/userdata   2019-07-30T07:31:54.768990081Z   root
fbe5b37ad3c472ea970af75afc4c58481c2dd4d89a93b1a7ca37ddda823b201c   31785       running     /run/containers/storage/overlay-containers/fbe5b37ad3c472ea970af75afc4c58481c2dd4d89a93b1a7ca37ddda823b201c/userdata   2019-07-30T07:34:15.524324699Z   root

# 首先确认节点情况
kubectl get nodes

# 然后告诉 API Server 暂停该节点的调度
kubectl drain <node name>

# 等节点恢复正常后，需要加回，可以用
kubectl uncordon <node name>

# Ubuntu 环境上用 apt-get 安装
apt install etcd-client

ETCD_VER=v3.5.4

# choose either URL
GOOGLE_URL=https://storage.googleapis.com/etcd
GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
DOWNLOAD_URL=${GOOGLE_URL}

rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test

curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz

/tmp/etcd-download-test/etcd --version
/tmp/etcd-download-test/etcdctl version
/tmp/etcd-download-test/etcdutl version
# start a local etcd server
/tmp/etcd-download-test/etcd

# write,read to etcd
/tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 put foo bar
/tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 get foo

root@ckalab001:~# ps -ef | grep api | grep -i etcd
root       24761   24743  3 10:17 ?        00:06:53 kube-apiserver --advertise-address=172.31.43.206 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key

# 设置环境变量，ETCDCTL_API=3
root@ckalab001:~# export ETCDCTL_API=3

root@ckalab001:~# etcdctl --cert="/etc/kubernetes/pki/apiserver-etcd-client.crt" --key="/etc/kubernetes/pki/apiserver-etcd-client.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints=https://127.0.0.1:2379 put /firstkey trystack
OK

root@ckalab001:~# etcdctl --cert="/etc/kubernetes/pki/apiserver-etcd-client.crt" --key="/etc/kubernetes/pki/apiserver-etcd-client.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints=https://127.0.0.1:2379 get /firstkey
/firstkey
trystack

# list 所有的 key
etcdctl --cert="/etc/kubernetes/pki/apiserver-etcd-client.crt" --key="/etc/kubernetes/pki/apiserver-etcd-client.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints=https://127.0.0.1:2379 get --prefix --keys-only ""

# list 所有的 key & value
etcdctl --cert="/etc/kubernetes/pki/apiserver-etcd-client.crt" --key="/etc/kubernetes/pki/apiserver-etcd-client.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints=https://127.0.0.1:2379 get --prefix ""

# backup & restore
etcdctl --cert="/etc/kubernetes/pki/apiserver-etcd-client.crt" --key="/etc/kubernetes/pki/apiserver-etcd-client.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints=https://127.0.0.1:2379 snapshot save a.txt
# etcdctl --cert="/etc/kubernetes/pki/apiserver-etcd-client.crt" --key="/etc/kubernetes/pki/apiserver-etcd-client.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints=https://127.0.0.1:2379 snapshot restore a.txt

apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: backup
  namespace: kube-system
spec:
  schedule: "0 0 * * *" # 指定时间
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            # Same image as in /etc/kubernetes/manifests/etcd.yaml
            image: k8s.gcr.io/etcd-amd64:3.2.18
            env:
            - name: ETCDCTL_API
              value: "3"
            command: ["/bin/sh"]
            args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /backup/etcd-snapshot-$(date +%Y-%m-%d_%H:%M:%S_%Z).db"]
            volumeMounts:
            - mountPath: /etc/kubernetes/pki/etcd
              name: etcd-certs
              readOnly: true
            - mountPath: /backup
              name: backup
          restartPolicy: OnFailure
          # 指定master0节点执行
          nodeName: master0
          hostNetwork: true
          volumes:
          - name: etcd-certs
            hostPath:
              path: /etc/kubernetes/pki/etcd
              type: DirectoryOrCreate
          - name: backup
            hostPath:
              path: /tmp/etcd_backup/
              type: DirectoryOrCreate

# 1. 将 /etc/kubernetes/manifests/ kube-apiserver.yaml 文件移动到别处，停止 kube-apiserver 服务
mv /etc/kubernetes/manifests/kube-apiserver.yaml /tmp/

# 2. 将 /etc/kubernetes/manifests/ etcd.yaml 文件移动到别处，停止 etcd server 服务
mv /etc/kubernetes/manifests/etcd.yaml /tmp/

# 3. 运行如下命令，将损坏的数据文件移至其他地方
mv /var/lib/etcd/* /tmp/

# 4. 运行以下命令，临时运行 docker 容器，将数据从备份里恢复到 /var/lib/etcd/
docker run --rm \
   -v '/tmp/etcd_backup/:/backup' \
   -v '/var/lib/etcd:/var/lib/etcd' \
   --env ETCDCTL_API=3 \
   'k8s.gcr.io/etcd-amd64:3.2.18' \
  /bin/sh -c "etcdctl snapshot restore '/backup/etcd-snapshot-xxx_UTC.db' ; mv /default.etcd/member/ /var/lib/etcd/"

# 5. 将/etc/kubernetes/manifests/kube-apiserver.yaml文件放回原来路径，恢复kube-api server服务
# 6. 将/etc/kubernetes/manifests/etcd.yaml文件放回原来路径，恢复etcd server服务

apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: k8s-master-backup
  namespace: kube-system
spec:
  # activeDeadlineSeconds: 100
  schedule: "5 0 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: k8s-master-backup
            image: docker.io/alpine:latest
            command: ["/bin/sh"]
            args: ["-c", "tar -zcvf /backup/k8s-master-$(ifconfig eth0 | grep 'inet addr:' | awk '{print $2}' | cut -c 6-)-$(date +%Y-%m-%d_%H:%M:%S_%Z).tar.gz /kubernetes /kubelet"]
            volumeMounts:
            - mountPath: /backup
              name: backup
            - mountPath: /kubernetes
              name: kubernetes
            - mountPath: /kubelet
              name: kubelet
          restartPolicy: OnFailure
          # 指定节点master0执行
          nodeName: master0
          hostNetwork: true
          volumes:
          - name: backup
            hostPath:
              path: /tmp/k8s_master_backup/
              type: DirectoryOrCreate
          - name: kubernetes
            hostPath:
              path: /etc/kubernetes/
              type: DirectoryOrCreate
          - name: kubelet
            hostPath:
              path: /var/lib/kubelet/
              type: DirectoryOrCreate

#!/bin/bash
scp -i .ssh/id_rsa /tmp/etcd_backup/ root@10.0.0.76:/tmp
scp -i .ssh/id_rsa /tmp/k8s_master_backup/ root@10.0.0.76:/tmp

# 加入脚本执行到计划任务
crontab -e

# 添加计划任务(计划每日0时30分执行，时间用户可自指定)
30 0 * * * /tmp/filescp.sh

wget -c https://sealyun-home.oss-cn-beijing.aliyuncs.com/sealos-4.0/latest/sealos-amd64 -O sealos && chmod +x sealos && mv sealos /usr/bin

sealos run labring/kubernetes:v1.24.0 labring/calico:v3.22.1 --masters 172.19.30.106

[root@lab-k8s001 ~]# crictl images
IMAGE                                       TAG                 IMAGE ID            SIZE
sealos.hub:5000/calico/cni                  v3.22.1             2a8ef6985a3e5       80.5MB
sealos.hub:5000/calico/node                 v3.22.1             7a71aca7b60fc       69.6MB
sealos.hub:5000/calico/pod2daemon-flexvol   v3.22.1             17300d20daf93       8.46MB
sealos.hub:5000/calico/typha                v3.22.1             f822f80398b9a       52.7MB
sealos.hub:5000/coredns/coredns             v1.8.6              a4ca41631cc7a       13.6MB
sealos.hub:5000/etcd                        3.5.3-0             aebe758cef4cd       102MB
sealos.hub:5000/kube-apiserver              v1.24.0             529072250ccc6       33.8MB
sealos.hub:5000/kube-controller-manager     v1.24.0             88784fb4ac2f6       31MB
sealos.hub:5000/kube-proxy                  v1.24.0             77b49675beae1       39.5MB
sealos.hub:5000/kube-scheduler              v1.24.0             e3ed7dee73e93       15.5MB
sealos.hub:5000/pause                       3.7                 221177c6082a8       311kB
sealos.hub:5000/tigera/operator             v1.25.3             648350e58702c       44.8MB

# 配置好密钥，确定可以 ssh 免密登录需要安装 kubelet 的 node

yum install -y openssl openssl-devel
yum install -y socat
yum install -y epel-release
yum install -y conntrack-tools

export KKZONE=cn
curl -sfL https://get-kk.kubesphere.io | VERSION=v2.0.0 sh -

./kk create config --with-kubernetes v1.21.5

vi config-sample.yaml

# {name: node1, address: 172.19.30.105, internalAddress: 172.19.30.105, user: root}
# 这里 172.19.30.105 是 node IP
# node1 要写你期望的节点名称，安装时会根据这个设置 hostname 的

./kk create cluster -f config-sample.yaml

# 部署完成后，检查 pod
kubectl get pod -A
docker ps

yum install -y yum-utils
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum install docker-ce-19.03.9 -y
mkdir -p /etc/docker
echo '{"registry-mirrors": ["http://hub-mirror.c.163.com"]}'>/etc/docker/daemon.json
systemctl enable --now docker

# curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--docker" sh -
# 受限国内网络，大多数时候上述脚本无法安装，建议采用下述国内加速安装脚本
curl -sfL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn INSTALL_K3S_EXEC="--docker" sh -

curl -sfL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -

...
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Skipping /usr/local/bin/ctr symlink to k3s, command exists in PATH at /usr/bin/ctr
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]  systemd: Enabling k3s unit
[INFO]  systemd: Starting k3s

$ kubectl get po -A
NAMESPACE     NAME                                      READY   STATUS      RESTARTS   AGE
kube-system   helm-install-traefik-crd-j4bwn            0/1     Completed   0          2m10s
kube-system   helm-install-traefik-sxnmw                0/1     Completed   0          2m10s
kube-system   metrics-server-86cbb8457f-47kzz           1/1     Running     0          2m10s
kube-system   local-path-provisioner-5ff76fc89d-7bzjd   1/1     Running     0          2m10s
kube-system   coredns-7448499f4d-cjph9                  1/1     Running     0          2m10s
kube-system   svclb-traefik-8q77r                       2/2     Running     0          83s
kube-system   traefik-97b44b794-gb72h                   1/1     Running     0          83s

# 集群基础使用
# 我们来创建一个 deployment 的 pod
$ kubectl create deployment nginx --image=nginx
$ kubectl get po
NAME                     READY   STATUS    RESTARTS   AGE
nginx-6799fc88d8-pc4wz   1/1     Running   0          101s

# 修改 CNI 网络模式
# 默认情况下，K3S 将以 flannel 作为 CNI 运行，使用 VXLAN 作为默认后端。如需修改 CNI 的模式，可以参考下述命令安装
# --flannel-backend=vxlan   (默认) 使用 VXLAN 后端。
# --flannel-backend=ipsec	使用 IPSEC 后端，对网络流量进行加密。
# --flannel-backend=host-gw	使用 host-gw 后端。
# --flannel-backend=wireguard	使用 WireGuard 后端，对网络流量进行加密。可能需要额外的内核模块和配置。
$ curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn INSTALL_K3S_EXEC="--docker --flannel-backend=vxlan" sh -

# 查看对应的 linux 网络设备是否生成
$ ip a | grep flannel

31: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default
    inet 10.42.0.0/32 scope global flannel.1

# 替换 CNI
# K3S 除了支持 flannel，还支持 calico\canel 等 CNI 组件
# 我们使用 calico 作为网络组件

# 注：下述 10.96.0.0/16 不能跟实际网络冲突
curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--flannel-backend=none --cluster-cidr=10.96.0.0/16 --disable-network-policy --disable=traefik" sh -

# 下载 calico.yaml
$ wget https://docs.projectcalico.org/manifests/calico.yaml
# 修改 calico.yaml 中被注释掉的参数 CALICO_IPV4POOL_CIDR
- name: CALICO_IPV4POOL_CIDR
  value: "10.96.0.0/16"

# apply calico
kubectl apply -f calico.yaml

# 查看集群 pod
$ kubectl get po -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
kube-system   calico-node-6cv7v                          1/1     Running   0          3m30s
kube-system   local-path-provisioner-5ff76fc89d-sqdts    1/1     Running   0          33m
kube-system   calico-kube-controllers-74b8fbdb46-mdwjm   1/1     Running   0          3m30s
kube-system   metrics-server-86cbb8457f-5xgrs            1/1     Running   0          33m
kube-system   coredns-7448499f4d-mwtmw                   1/1     Running   0          33m

# 要从 server 节点卸载 K3s
$ /usr/local/bin/k3s-uninstall.sh

# 要从 agent 节点卸载 K3s
$ /usr/local/bin/k3s-agent-uninstall.sh

# 所有节点均部署 docker，与上述单节点部署过程一样，不再赘述
# etcd 高可用部署
# 三台 master 节点上均执行下述的 etcd 操作流程

wget https://github.com/etcd-io/etcd/releases/download/v3.3.15/etcd-v3.3.15-linux-amd64.tar.gz $
tar -vxf etcd-v3.3.15-linux-amd64.tar.gz $ cp etcd-v3.3.15-linux-amd64/etcd /usr/bin/ $ cp
etcd-v3.3.15-linux-amd64/etcdctl /usr/bin/

# etcd 采用 systemd 托管服务

vi /etc/systemd/system/etcd.service

# 填入 etcd 配置，保存退出。（etcd.service 配置见附录）

$ systemct start etcd $ systemct status etcd ● etcd.service - etcd Loaded: loaded
(/etc/systemd/system/etcd.service; disabled; vendor preset: disabled) Active: active (running) since
Sun 2021-10-03 18:19:52 CST; 10s ago Docs: https://github.com/etcd-io/etcd

$ etcdctl member list member 763586516b512018 is healthy: got healthy result from
http://{etcd1-ip}:2379 member 78bb25a565876d17 is healthy: got healthy result from
http://{etcd2-ip}:2379 member da51b2723088b522 is healthy: got healthy result from
http://{etcd3-ip}:2379

systemctl stop etcd
rm -rf /var/lib/etcd
systemctl restart etcd

$ curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn K3S_DATASTORE_ENDPOINT='http://{etcd1-ip}:2379,http://{etcd2-ip}:2379,http://{etcd3-ip}:2379' K3S_TOKEN="k3s_token" INSTALL_K3S_VERSION=v1.18.6+k3s1 INSTALL_K3S_EXEC="--docker" sh -s - server
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Creating /usr/local/bin/ctr symlink to k3s
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]  systemd: Enabling k3s unit
[INFO]  systemd: Starting k3s

$ kubectl get no
NAME         STATUS   ROLES                  AGE     VERSION
caasnode-1   Ready    control-plane,master   2m58s   v1.21.4+k3s1
caasnode-2   Ready    control-plane,master   78s     v1.21.4+k3s1
caasnode-3   Ready    control-plane,master   82s     v1.21.4+k3s1

$ curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn K3S_TOKEN="k3s_token" K3S_URL=https://{master-ip}:6443 INSTALL_K3S_VERSION=v1.18.6+k3s1 INSTALL_K3S_EXEC="--docker" sh -
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Skipping /usr/local/bin/ctr symlink to k3s, command exists in PATH at /usr/bin/ctr
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-agent-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s-agent.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s-agent.service
[INFO]  systemd: Enabling k3s-agent unit
[INFO]  systemd: Starting k3s-agent

$ kubectl get no
caasbastion-3   Ready    <none>                 3m20s   v1.21.4+k3s1
caasbastion-2   Ready    <none>                 2m49s   v1.21.4+k3s1
caasbastion-1   Ready    <none>                 2m27s   v1.21.4+k3s1
caasnode-1      Ready    control-plane,master   21m     v1.21.4+k3s1
caasnode-2      Ready    control-plane,master   19m     v1.21.4+k3s1
caasnode-3      Ready    control-plane,master   19m     v1.21.4+k3s1

# 要从 server 节点卸载 K3s
$ /usr/local/bin/k3s-uninstall.sh

# 要从 agent 节点卸载 K3s
$ /usr/local/bin/k3s-agent-uninstall.sh

# 如需重新部署，需要清理 etcd 的 data 目录，文档默认为 '/var/lib/etcd'

# /etc/systemd/system/etcd.service
# etcd-ip: 本机 ip
# etcd1-ip: 本机 ip
# etcd2-ip: 第二台节点 ip
# etcd3-ip: 第三台节点 ip
[Unit]
Description=etcd
Documentation=https://github.com/etcd-io/etcd
After=network-online.target

[Service]
Type=notify
Restart=on-failure
RestartSec=5s
TimeoutStartSec=0
ExecStart=/usr/bin/etcd --name etcd1 \
--heartbeat-interval 300 \
--election-timeout 1500 \
--initial-advertise-peer-urls http://{etcd-ip}:2380 \
--listen-peer-urls http://{etcd-ip}:2380 \
--listen-client-urls http://{etcd-ip}:2379,http://127.0.0.1:2379 \
--advertise-client-urls http://{etcd-ip}:2379 \
--data-dir /var/lib/etcd \
--initial-cluster etcd1=http://{etcd1-ip}:2380,etcd2=http://{etcd2-ip}:2380,etcd3=http://{etcd3-ip}:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new \
--auto-compaction-retention=1 \
--snapshot-count=5000  \
--quota-backend-bytes=171798691840 \
--heartbeat-interval=100 \
--election-timeout=500
ExecReload=/bin/kill -HUP
KillMode=process

[Install]
WantedBy=multi-user.target

curl -sS https://rancher-mirror.oss-cn-beijing.aliyuncs.com/autok3s/install.sh | INSTALL_AUTOK3S_MIRROR=cn sh -

# The commands will start autok3s daemon with an interactionable UI.
autok3s -d serve --bind-address 0.0.0.0

alias k=kubectl

k drain cluster3-master1 --ignore-daemonsets
apt-mark unhold kubeadm
apt-mark hold kubectl kubelet
apt install kubeadm=1.23.1-00
apt-mark hold kubeadm

kubeadm upgrade plan
kubeadm upgrade apply v1.23.1

apt update
apt-mark unhold kubelet kubectl
apt install kubelet=1.23.1-00 kubectl=1.23.1-00
apt-mark hold kubelet kubectl
service kubelet restart
service kubelet status
kubectl get node

k uncordon cluster3-master1

k drain cluster3-worker1 --ignore-daemonsets

apt update
apt-mark unhold kubeadm
apt-mark hold kubectl kubelet
apt install kubeadm=1.23.1-00
apt-mark hold kubeadm
kubeadm upgrade node

apt-mark unhold kubectl kubelet
apt install kubelet=1.23.1-00 kubectl=1.23.1-00
service kubelet restart
service kubelet status
k get node
k uncordon cluster3-worker1

yum list --showduplicates kubeadm --disableexcludes=kubernetes
# find the latest 1.23 version in the list
# it should look like 1.23.x-0, where x is the latest patch

# replace x in 1.23.x-0 with the latest patch version
yum install -y kubeadm-1.23.x-0 --disableexcludes=kubernetes

# Verify that the download works and has the expected version:
kubeadm version

# Verify the upgrade plan:
kubeadm upgrade plan

# replace x with the patch version you picked for this upgrade
sudo kubeadm upgrade apply v1.23.x

# Same as the first control plane node but use:
sudo kubeadm upgrade node
# instead of: sudo kubeadm upgrade apply
# Also calling "kubeadm upgrade plan" and upgrading the CNI provider plugin is no longer needed.

# replace <node-to-drain> with the name of your node you are draining
kubectl drain <node-to-drain> --ignore-daemonsets

# replace x in 1.23.x-0 with the latest patch version
yum install -y kubelet-1.23.x-0 kubectl-1.23.x-0 --disableexcludes=kubernetes

# Restart the kubelet:
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# replace <node-to-drain> with the name of your node
kubectl uncordon <node-to-drain>

$ docker run -d --restart=unless-stopped -p 1234:80 -p 2234:443 rancher/rancher:v2.3.6

sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent:v2.3.6 --server https://172.20.149.95:2234 --token ndhsgr9msksr4s6g6xqq6dl8cns42txlkxt96tsdwcl56hds568nbb --ca-checksum 5b778f411871d7caca521a0cf25cd40d8a7e28aeb1def47af829e22092c114cd --etcd --controlplane --worker

[root@lab-c2009-ceph-aio ~]# uname -r
5.4.182-1.el7.elrepo.x86_64
[root@lab-c2009-ceph-aio ~]# cat /etc/system-release
CentOS Linux release 7.9.2009 (Core)

[root@lab-c2009-ceph-aio ~]# yum install python3 -y
[root@lab-c2009-ceph-aio ~]# python3 -V
Python 3.6.8

[root@lab-c2009-ceph-aio ~]# ls /dev/vd*
/dev/vda  /dev/vda1  /dev/vda2  /dev/vdb

yum install -y yum-utils
yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo

yum install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin

yum list docker-ce --showduplicates | sort -r

yum install -y docker-ce-20.10.16 docker-ce-cli-20.10.16 containerd.io docker-compose-plugin

systemctl enable docker --now

docker version

curl -k --remote-name --location https://github.com/ceph/ceph/raw/octopus/src/cephadm/cephadm
chmod +x cephadm

# ./cephadm add-repo --release octopus
# 报错：repomd.xml: [Errno 14] curl#60 - "The certificate issuer‘s certificate has expired"
# 参考：https://blog.csdn.net/sulia1234567890/article/details/121956448
# 加入 vi /etc/yum.conf，增加：sslverify=0
vi /etc/yum.conf # sslverify=0
yum upgrade ca-certificates
./cephadm add-repo --release octopus
./cephadm install

which cephadm
# /usr/sbin/cephadm

[root@lab-c2009-ceph-aio ~]# mkdir -p /etc/ceph
[root@lab-c2009-ceph-aio ~]# ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:cd:72:83 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.229/24 brd 192.168.122.255 scope global noprefixroute dynamic eth0
       valid_lft 2736sec preferred_lft 2736sec
    inet6 fe80::5390:413f:f60f:6034/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

cat <<EOF > initial-ceph.conf
[global]
osd pool default size = 1
osd pool default min size = 1
EOF

cephadm bootstrap --config initial-ceph.conf --mon-ip 192.168.122.229

# Ceph Dashboard is now available at:

# 	     URL: https://lab-c2009-ceph-aio:8443/
# 	    User: admin
# 	Password: itgwv8r3xy

# You can access the Ceph CLI with:

# 	sudo /usr/sbin/cephadm shell --fsid 090a76ec-9bb4-11ec-91af-525400cd7283 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring

[root@lab-c2009-ceph-aio ~]# cephadm add-repo --release octopus
[root@lab-c2009-ceph-aio ~]# yum install -y ceph-common
[root@lab-c2009-ceph-aio ~]# ceph -s
  cluster:
    id:     090a76ec-9bb4-11ec-91af-525400cd7283
    health: HEALTH_WARN
            OSD count 0 < osd_pool_default_size 1

  services:
    mon: 1 daemons, quorum lab-c2009-ceph-aio (age 58m)
    mgr: lab-c2009-ceph-aio.nuvwkm(active, since 57m)
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:

[root@lab-c2009-ceph-aio ~]# ceph health
HEALTH_WARN OSD count 0 < osd_pool_default_size 1

[root@lab-c2009-ceph-aio ~]# ceph orch device ls
Hostname            Path      Type  Serial  Size   Health   Ident  Fault  Available
lab-c2009-ceph-aio  /dev/vdb  hdd           21.4G  Unknown  N/A    N/A    Yes

[root@lab-c2009-ceph-aio ~]# ceph orch apply osd --all-available-devices
Scheduled osd.all-available-devices update...

[root@lab-c2009-ceph-aio ~]# ceph orch daemon add osd lab-c2009-ceph-aio:/dev/vdb
Created no osd(s) on host lab-c2009-ceph-aio; already created?

[root@lab-c2009-ceph-aio ~]# ceph -s
  cluster:
    id:     090a76ec-9bb4-11ec-91af-525400cd7283
    health: HEALTH_WARN
            1 pool(s) have no replicas configured

  services:
    mon: 1 daemons, quorum lab-c2009-ceph-aio (age 13m)
    mgr: lab-c2009-ceph-aio.fgkrqf(active, since 12m)
    osd: 1 osds: 1 up (since 2m), 1 in (since 2m)

  data:
    pools:   1 pools, 1 pgs
    objects: 0 objects, 0 B
    usage:   1.0 GiB used, 19 GiB / 20 GiB avail
    pgs:     1 active+clean

[root@lab-c2009-ceph-aio ~]# ceph osd pool create kubernetes
pool 'kubernetes' created

[root@lab-c2009-ceph-aio ~]# ceph osd lspools
1 device_health_metrics
2 kubernetes

[root@lab-c2009-ceph-aio ~]# ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=kubernetes' mgr 'profile rbd pool=kubernetes'
[client.kubernetes]
	key = AQBaByJiJbL8KRAAW77WOh1hu6Pz9if9EPj9mA==

[root@lab-c2009-ceph-aio ~]# ceph auth get client.kubernetes
exported keyring for client.kubernetes
[client.kubernetes]
	key = AQBaByJiJbL8KRAAW77WOh1hu6Pz9if9EPj9mA==
	caps mgr = "profile rbd pool=kubernetes"
	caps mon = "profile rbd"
	caps osd = "profile rbd pool=kubernetes"

yum install wget -y
wget --no-check-certificate https://github.com/ceph/ceph-csi/archive/refs/tags/v3.6.1.tar.gz
tar zxvf v3.6.1.tar.gz

[root@lab-c2009-ceph-aio ~]# ceph mon dump
dumped monmap epoch 1
epoch 1
fsid 090a76ec-9bb4-11ec-91af-525400cd7283
last_changed 2022-03-04T12:12:10.455157+0000
created 2022-03-04T12:12:10.455157+0000
min_mon_release 15 (octopus)
0: [v2:192.168.122.229:3300/0,v1:192.168.122.229:6789/0] mon.lab-c2009-ceph-aio

apiVersion: v1
kind: ConfigMap
metadata:
  name: "ceph-csi-config"
data:
  config.json: |-
    [
      {
        "clusterID": "090a76ec-9bb4-11ec-91af-525400cd7283",
        "monitors": [
          "192.168.122.229:6789"
        ]
      }
    ]

cd /root/ceph-csi-3.6.1/deploy/rbd/kubernetes

kubectl apply -f csi-config-map.yaml

cat <<EOF > csi-rbd-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: csi-rbd-secret
stringData:
  userID: kubernetes
  userKey: AQBaByJiJbL8KRAAW77WOh1hu6Pz9if9EPj9mA==
EOF

kubectl apply -f csi-rbd-secret.yaml

kubectl create -f csi-provisioner-rbac.yaml
kubectl create -f csi-nodeplugin-rbac.yaml

kubectl create -f csi-provisioner-psp.yaml
kubectl create -f csi-nodeplugin-psp.yaml

diff kubernetes-bak/csi-rbdplugin-provisioner.yaml kubernetes/csi-rbdplugin-provisioner.yaml
160,161c160,161
<             - name: ceph-csi-encryption-kms-config
<               mountPath: /etc/ceph-csi-encryption-kms-config/
---
>             # - name: ceph-csi-encryption-kms-config
>             #   mountPath: /etc/ceph-csi-encryption-kms-config/
230,232c230,232
<         - name: ceph-csi-encryption-kms-config
<           configMap:
<             name: ceph-csi-encryption-kms-config
---
>         # - name: ceph-csi-encryption-kms-config
>         #   configMap:
>         #     name: ceph-csi-encryption-kms-config

diff kubernetes-bak/csi-rbdplugin.yaml kubernetes/csi-rbdplugin.yaml
107,108c107,108
<             - name: ceph-csi-encryption-kms-config
<               mountPath: /etc/ceph-csi-encryption-kms-config/
---
>             # - name: ceph-csi-encryption-kms-config
>             #   mountPath: /etc/ceph-csi-encryption-kms-config/
188,190c188,190
<         - name: ceph-csi-encryption-kms-config
<           configMap:
<             name: ceph-csi-encryption-kms-config
---
>         # - name: ceph-csi-encryption-kms-config
>         #   configMap:
>         #     name: ceph-csi-encryption-kms-config

[root@lab-c2009-k8s-aio-ceph kubernetes]# kubectl apply -f ../../../examples/ceph-conf.yaml

[root@lab-c2009-k8s-aio-ceph kubernetes]# kubectl get cm -A
NAMESPACE         NAME                                 DATA   AGE
ceph-csi          ceph-config                          2      27s

kubectl create -f csi-rbdplugin-provisioner.yaml
kubectl create -f csi-rbdplugin.yaml

kubectl scale deploy csi-rbdplugin-provisioner --replicas=1
kubectl taint nodes $(hostname) node-role.kubernetes.io/master:NoSchedule-

cat <<EOF > storageclass.yaml
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
   clusterID: 090a76ec-9bb4-11ec-91af-525400cd7283
   pool: kubernetes
   imageFeatures: layering
   csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
   csi.storage.k8s.io/provisioner-secret-namespace: default
   csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
   csi.storage.k8s.io/controller-expand-secret-namespace: default
   csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
   csi.storage.k8s.io/node-stage-secret-namespace: default
   csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
   - discard
EOF

kubectl apply -f storageclass.yaml

[root@lab-c2009-k8s-aio-ceph rbd]# kubectl apply -f pvc.yaml
persistentvolumeclaim/rbd-pvc created

[root@lab-c2009-k8s-aio-ceph rbd]# kubectl get pvc
NAMESPACE   NAME      STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
default     rbd-pvc   Pending                                      csi-rbd-sc     7s

[root@lab-c2009-k8s-aio-ceph rbd]# kubectl get pvc
NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
rbd-pvc   Bound    pvc-d45b457f-9c67-4b82-b28b-329a1eb1747d   1Gi        RWO            csi-rbd-sc     4m30s

[root@lab-c2009-k8s-aio-ceph rbd]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS   REASON   AGE
pvc-d45b457f-9c67-4b82-b28b-329a1eb1747d   1Gi        RWO            Delete           Bound    default/rbd-pvc   csi-rbd-sc              1s

[root@lab-c2009-ceph-aio ~]# rbd ls -p kubernetes
csi-vol-7b95cc49-9c1f-11ec-855a-ea06f4beb16f

[root@lab-c2009-ceph-aio ~]# rbd info csi-vol-7b95cc49-9c1f-11ec-855a-ea06f4beb16f -p kubernetes
rbd image 'csi-vol-7b95cc49-9c1f-11ec-855a-ea06f4beb16f':
	size 1 GiB in 256 objects
	order 22 (4 MiB objects)
	snapshot_count: 0
	id: 37b4396277f1
	block_name_prefix: rbd_data.37b4396277f1
	format: 2
	features: layering
	op_features:
	flags:
	create_timestamp: Fri Mar  4 19:59:19 2022
	access_timestamp: Fri Mar  4 19:59:19 2022
	modify_timestamp: Fri Mar  4 19:59:19 2022

[root@lab-c2009-k8s-aio-ceph rbd]# kubectl apply -f pod.yaml

[root@lab-c2009-k8s-aio-ceph rbd]# lsblk -l|grep rbd
rbd0        251:0    0    1G  0 disk /var/lib/kubelet/pods/9f47c56f-675c-40fc-9b6e-5f00676bf8d0/volumes/kubernetes.io~csi/pvc-d45b457f-9c67-4b82-b28b-329a1eb1747d/mount

[root@lab-c2009-k8s-aio-ceph rbd]# kubectl exec -it csi-rbd-demo-pod -- bash

root@csi-rbd-demo-pod:/# lsblk -l|grep rbd
rbd0 251:0    0    1G  0 disk /var/lib/www/html

[root@lab-k8s004 ~]# kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.22/deploy/local-path-storage.yaml
namespace/local-path-storage created
serviceaccount/local-path-provisioner-service-account created
clusterrole.rbac.authorization.k8s.io/local-path-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/local-path-provisioner-bind created
deployment.apps/local-path-provisioner created
storageclass.storage.k8s.io/local-path created
configmap/local-path-config created

[root@lab-k8s004 ~]# kubectl get sc
NAME         PROVISIONER             RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
local-path   rancher.io/local-path   Delete          WaitForFirstConsumer   false                  50s

[root@lab-k8s004 ~]# kubectl patch storageclass local-path -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class":"true"}}}'
storageclass.storage.k8s.io/local-path patched

[root@lab-k8s004 ~]# kubectl get sc
NAME                   PROVISIONER             RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
local-path (default)   rancher.io/local-path   Delete          WaitForFirstConsumer   false                  75s

[root@lab-k8s004 ~]# kubectl create -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/examples/pvc/pvc.yaml
persistentvolumeclaim/local-path-pvc created

[root@lab-k8s004 ~]# kubectl create -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/examples/pod/pod.yaml
pod/volume-test created

[root@lab-k8s004 ~]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                    STORAGECLASS   REASON   AGE
pvc-013cdb6b-fe6e-4fa8-9e5b-8a5e4957a91b   128Mi      RWO            Delete           Bound    default/local-path-pvc   local-path              23s

[root@lab-k8s004 ~]# kubectl get pvc
NAME             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
local-path-pvc   Bound    pvc-013cdb6b-fe6e-4fa8-9e5b-8a5e4957a91b   128Mi      RWO            local-path     40s

echo '{"cniVersion": "0.3.1","name": "mynet","type": "macvlan","bridge": "cni0","isGateway": true,"ipMasq": true,"ipam": {"type": "host-local","subnet": "10.244.1.0/24","routes": [{ "dst": "0.0.0.0/0" }]}}' | sudo CNI_COMMAND=ADD CNI_NETNS=/var/run/netns/a CNI_PATH=./bin CNI_IFNAME=eth0 CNI_CONTAINERID=a CNI_VERSION=0.3.1 ./bin/bridge

echo '{"cniVersion": "0.3.1","type":"IGNORED", "name": "a","ipam": {"type": "host-local", "subnet":"10.1.2.3/24"}}' | sudo CNI_COMMAND=ADD CNI_NETNS=/var/run/netns/a CNI_PATH=./bin CNI_IFNAME=a CNI_CONTAINERID=a CNI_VERSION=0.3.1 ./bin/host-local

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: test-network-policy
  namespace: default
spec:
  # podSelector: {} 表示选择所有 pod 应用 NetworkPolicy
  podSelector: # 表示选择包含标签 role=db 的 pod 应用下面的 NetworkPolicy
    matchLabels:
      role: db
  policyTypes: # 表示 NetworkPolicy 包含 ingress 和 egress 流量规则
  - Ingress
  - Egress
  ingress: # ingress 规则白名单列表，每条规则允许同时匹配 from 和 ports  流，可以有条个规则。
  # 第 1 条白名单，包含 from + ports 的组合规则，允许来自172.17网段(172.17.1除外)、或标 签 project=myproject 的命名空间的所 有 pod 、或 default 命名空间下标签 role=frontend 的 pod 访问（限 tcp 6379 端口）
  - from:
    - ipBlock:
        cidr: 172.17.0.0/16
        except:
        - 172.17.1.0/24
    - namespaceSelector:
        matchLabels:
          project: myproject
    - podSelector:
        matchLabels:
          role: frontend
    ports:
    - protocol: TCP
      port: 6379
  # 第二条白名单，只包含 from 规则，允许来自所有命名空间包含 environment=testing 标签的 pod 访问（不限端口）
  - from:
    - namespaceSelector: {}
      podSelector:
        matchLabels:
          environment: testing
  egress: # egress 规则白名单列表，同 ingress 规则一样，每条规则包含 to+ports，可以有多条规则。
  - to:
    - ipBlock:
        cidr: 10.0.0.0/24
    ports:
    - protocol: TCP
      port: 5978

apiVersion: v1
kind: Pod
metadata:
  name: podx
  annotations:
    container.apparmor.security.beta.kubernetes.io/podx: localhost/nginx-profile-3
spec:
  containers:
  - image: busybox
    imagePullPolicy: IfNotPresent
    name: podx
    command: [ "sh", "-c", "echo 'Hello AppArmor!' && sleep 1h" ]
    resources: {}
  nodeName: node01
  dnsPolicy: ClusterFirst
  restartPolicy: Always

apiVersion: v1
kind: ServiceAccount
metadata:
  name: backend-sa
  namespace: qa
automountServiceAccountToken: false

apiVersion: v1
kind: Pod
metadata:
  name: backend
  namespace: qa
spec:
  serviceAccountName: backend-sa
  containers:
  - image: nginx:1.9
    imagePullPolicy: IfNotPresent
    name: backend

k -n restricted get pod -o yaml | grep automountServiceAccountToken

curl https://kubernetes.default/api/v1/namespaces/restricted/secrets -H "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" -k

$ kubectl apply -f https://github.com/aquasecurity/kube-bench/raw/main/job.yaml
job.batch/kube-bench created

$ kubectl get pods
NAME                      READY   STATUS              RESTARTS   AGE
kube-bench-j76s9   0/1     ContainerCreating   0          3s

# Wait for a few seconds for the job to complete
$ kubectl get pods
NAME                      READY   STATUS      RESTARTS   AGE
kube-bench-j76s9   0/1     Completed   0          11s

# The results are held in the pod's logs
kubectl logs kube-bench-j76s9
[INFO] 1 Master Node Security Configuration
[INFO] 1.1 API Server
...

kube-bench run --targets=master | grep kube-controller -A 3

kube-bench run --targets=node | grep /var/lib/kubelet/config.yaml -B2