diff --git a/README.en.md b/README.en.md index 7a367f27445fb09f36e24b1198c2c63b960fcfa0..29b30c26f8c419e6f476e0ea0509dd2772149f9c 100644 --- a/README.en.md +++ b/README.en.md @@ -9,7 +9,7 @@ Supporting the openEuler series of operating systems under the Kunpeng architect ## Instructions - [Remote attestation](https://gitee.com/openeuler/virtCCA_sdk/blob/master/README.md#%E8%BF%9C%E7%A8%8B%E8%AF%81%E6%98%8E) -- [Kata(cc0.8.0 version)](https://gitee.com/openeuler/virtCCA_sdk/blob/master/kata-cc0.8.0/doc/en-us/kata-confidential-containers.md) +- [Kata(cc0.8.0 version)](https://gitee.com/openeuler/virtCCA_sdk/blob/master/kata-cc0.8.0/doc/en-us/confidential_container/kata-confidential-containers.md) - [Kata(v3.15.0 version)](https://gitee.com/openeuler/virtCCA_sdk/blob/master/kata-v3.15.0/doc/kata%E6%9C%BA%E5%AF%86%E5%AE%B9%E5%99%A8.md) diff --git a/README.md b/README.md index d5c6d9738f5718b203c81b7bac387990638d05b0..40319a7291ed20fb23cb4cc61441f867b5158cb9 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ virtCCA(包含 TEE 虚拟化的 ARM 机密计算架构)的软件开发工具 ## 使用说明 - [远程证明](#远程证明) -- [kata(cc0.8.0版本)](https://gitee.com/openeuler/virtCCA_sdk/blob/master/kata-cc0.8.0/doc/zh-cn/kata%E6%9C%BA%E5%AF%86%E5%AE%B9%E5%99%A8.md) +- [kata(cc0.8.0版本)](https://gitee.com/openeuler/virtCCA_sdk/blob/master/kata-cc0.8.0/doc/zh-cn/confidential_container/kata%E6%9C%BA%E5%AF%86%E5%AE%B9%E5%99%A8.md) - [kata(v3.15.0版本)](https://gitee.com/openeuler/virtCCA_sdk/blob/master/kata-v3.15.0/doc/kata%E6%9C%BA%E5%AF%86%E5%AE%B9%E5%99%A8.md) **机密容器推荐使用此版本** - mpc使用样例 @@ -136,4 +136,4 @@ virtCCA(包含 TEE 虚拟化的 ARM 机密计算架构)的软件开发工具 ``` 如果您想为本仓库贡献代码,请向本仓库任意maintainer发送邮件 如果您找到产品中的任何Bug,欢迎您提出ISSUE -``` \ No newline at end of file +``` diff --git a/kata-v3.15.0/conf/pcipc-nic.json b/kata-v3.15.0/conf/pcipc-nic.json new file mode 100644 index 0000000000000000000000000000000000000000..ff54ed035caf9e09e1ea1fd4c141701265471a43 --- /dev/null +++ b/kata-v3.15.0/conf/pcipc-nic.json @@ -0,0 +1,16 @@ +{ + "cdiVersion": "0.6.0", + "kind": "pcipc/nic", + "devices": [ + { + "name": "1", + "containerEdits": { + "deviceNodes": [ + { + "path": "/dev/vfio/77" + } + ] + } + } + ] +} diff --git a/kata-v3.15.0/conf/pcipc-nvme-hook.sh b/kata-v3.15.0/conf/pcipc-nvme-hook.sh new file mode 100644 index 0000000000000000000000000000000000000000..7dfead13f2d457a885707ee2f0fc74e795d0bab9 --- /dev/null +++ b/kata-v3.15.0/conf/pcipc-nvme-hook.sh @@ -0,0 +1,95 @@ +#!/bin/sh + +# configuration params +DEVICE="/dev/nvme0n1" +MOUNT_POINT="/mnt" # global mount point +KATA_BASE="/run/kata-containers" +LOG_FILE="/tmp/prestart.log" +BIND_SUFFIX="pcipci_disk" + +# check the device mounting status +is_device_mounted() { + findmnt -n -o SOURCE "$DEVICE" >/dev/null 2>&1 && return 0 + return 1 +} + +# log function +log() { + local level=$1 + local message=$2 + local symbol="" + case "$level" in + "INFO") symbol="" ;; + "SUCCESS") symbol="" ;; + "WARN") symbol="" ;; + "ERROR") symbol="" ;; + esac + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $symbol [$level] $message" | tee -a "$LOG_FILE" +} + +if [ -e "/dev/nvme0n1" ]; then + log "INFO" "Found NVMe block device /dev/nvme0n1!" + + # step1 Global Mounting of NVMe Devices (Idempotent Operation) + if ! is_device_mounted; then + log "INFO" "mounting $DEVICE to $MOUNT_POINT" + mkdir -p "$MOUNT_POINT" + if mount "$DEVICE" "$MOUNT_POINT"; then + log "SUCCESS" "the device has been mounted to $MOUNT_POINT" + else + log "ERROR" "mount device failed, error code: $?" + exit 1 + fi + else + log "INFO" "the device has been mounted to $MOUNT_POINT, skip" + fi + + # step2 Accurately bind container directories + find "$KATA_BASE" -maxdepth 2 -type d -name "rootfs" | while read -r ROOTFS_DIR; do + + # Extract Container ID(e.g. extract a86ad41... from /run/kata-containers/a86ad41.../rootfs) + CONTAINER_ID=$(basename "$(dirname "$ROOTFS_DIR")") + + # Create a dedicated empty directory for the container + CONTAINER_MOUNT="$MOUNT_POINT/$CONTAINER_ID" + if [ ! -d "$CONTAINER_MOUNT" ]; then + if ! mkdir -p "$CONTAINER_MOUNT"; then + log "ERROR" "Create container directory failed: $CONTAINER_MOUNT" + continue + else + log "SUCCESS" "Create a container-specific directory: $CONTAINER_MOUNT" + fi + fi + + # Check if the exclusive directory is empty (security protection) + if [ "$(ls -A "$CONTAINER_MOUNT")" ]; then + log "WARN" "The directory is not empty! Clear it before mounting: $CONTAINER_MOUNT" + rm -rf $CONTAINER_MOUNT/* + fi + + # Calculate the bound target path (e.g. /run/kata-containers//rootfs/$BIND_SUFFIX) + BIND_TARGET="$ROOTFS_DIR/$BIND_SUFFIX" + + # Skip the mounted directory + if findmnt -n -o TARGET "$BIND_TARGET" >/dev/null; then + log "INFO" "Skip Already Mounted: $BIND_TARGET" + continue + fi + + # Create target directory (force mode) + if ! mkdir -p "$BIND_TARGET" 2>/dev/null; then + log "ERROR" "Create directory failed: $BIND_TARGET" + continue + fi + + # Perform binding and mounting. + if mount --bind "$CONTAINER_MOUNT" "$BIND_TARGET"; then + log "SUCCESS" "bind successfully: $CONTAINER_MOUNT $BIND_TARGET" + else + log "ERROR" "bind failed: $BIND_TARGET (error code: $?)" + fi + done +else + log "INFO" "NVMe block device /dev/nvme0n1 not found!" + exit 1 +fi \ No newline at end of file diff --git a/kata-v3.15.0/conf/pcipc-nvme.json b/kata-v3.15.0/conf/pcipc-nvme.json new file mode 100644 index 0000000000000000000000000000000000000000..f49967d33a46f5ceb4d29a52a10f58226b31bf33 --- /dev/null +++ b/kata-v3.15.0/conf/pcipc-nvme.json @@ -0,0 +1,16 @@ +{ + "cdiVersion": "0.6.0", + "kind": "pcipc/nvme", + "devices": [ + { + "name": "1", + "containerEdits": { + "deviceNodes": [ + { + "path": "/dev/vfio/42" + } + ] + } + } + ] +} diff --git a/kata-v3.15.0/conf/virtcca.config b/kata-v3.15.0/conf/virtcca.config index bd40d8b1bb2fdf2d29183f645cb7be478a6916f8..7a9c1e893066009873838dd6c81c227c1817fe23 100644 --- a/kata-v3.15.0/conf/virtcca.config +++ b/kata-v3.15.0/conf/virtcca.config @@ -1930,8 +1930,9 @@ CONFIG_ETHERNET=y # CONFIG_NET_VENDOR_HISILICON is not set CONFIG_NET_VENDOR_HUAWEI=y # CONFIG_HINIC is not set -# CONFIG_HINIC3 is not set +CONFIG_HINIC3=y # CONFIG_BMA is not set +# CONFIG_HIBIFUR is not set # CONFIG_NET_VENDOR_I825XX is not set CONFIG_NET_VENDOR_INTEL=y # CONFIG_E100 is not set diff --git "a/kata-v3.15.0/doc/\346\234\272\345\257\206\345\256\271\345\231\250\346\224\257\346\214\201SRIOV.md" "b/kata-v3.15.0/doc/\346\234\272\345\257\206\345\256\271\345\231\250\346\224\257\346\214\201SRIOV.md" index 2406a7d818c7f3c25eaec68ee73d0643b2e7fe3e..3d8f46d64e0d8103119f9d924fd624c69da932a6 100644 --- "a/kata-v3.15.0/doc/\346\234\272\345\257\206\345\256\271\345\231\250\346\224\257\346\214\201SRIOV.md" +++ "b/kata-v3.15.0/doc/\346\234\272\345\257\206\345\256\271\345\231\250\346\224\257\346\214\201SRIOV.md" @@ -54,14 +54,119 @@ ![](figures/zh-cn_image_0000002304611986.png) -3. 修改ctr容器 配置文件支持vfio设备冷插拔。 +> 前提条件:已参照**kata-deploy自动化部署**章节完成机密容器环境部署 +## ctr命令启动使能SRIOV的机密容器 +1. 修改ctr默认容器运行时配置文件支持vfio设备冷插拔。 `vim /etc/kata-containers/configuration.toml` - 添加:`cold_plug_vfio = "root-port"` + 在`hypervisor.qemu`标签下添加:`cold_plug_vfio = "root-port"` -4. ctr启动机密容器时通过--device透传vfio设备。 +2. ctr启动机密容器时通过--device透传vfio设备。 ctr run --runtime "io.containerd.kata.v2" --device /dev/vfio/91 --rm -t docker.io/library/busybox:latest kata-test /bin/sh ![](figures/zh-cn_image_0000002338371509.png) 容器中可以看到直通的VF设备ID。 + + +## k8s启动使能SRIOV的机密容器 +1. 修改`kata-qemu-virtcca`容器运行时配置文件支持vfio设备冷插拔。 + `vim /opt/kata/share/defaults/kata-containers/configuration-qemu-virtcca.toml` + 在`hypervisor.qemu`标签下添加:`cold_plug_vfio = "root-port"` + +2. 修改containerd配置文件以支持cdi设备注入的注解。 + `vim /etc/containerd/config.toml` + 在`.containerd.runtimes.kata-qemu-virtcca`标签下作如下修改: + - 新增:`privileged_without_host_devices_all_devices_allowed = true` + - pod_annotations中新增:`"cdi.k8s.io/vfio*"` + + 完成修改后的内容如下: +```shell +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-virtcca] +runtime_type = "io.containerd.kata-qemu.v2" +runtime_path = "/opt/kata/bin/containerd-shim-kata-v2" +privileged_without_host_devices = true +privileged_without_host_devices_all_devices_allowed = true +pod_annotations = ["io.katacontainers.*", "cdi.k8s.io/vfio*"] +``` +`systemctl daemon-reload && systemctl restart containerd` 使配置生效。 + +3. 新增cdi设备注入配置文件 +```shell +mkdir -p /etc/cdi +cp ./virtCCA_sdk/kata-v3.15.0/conf/pcipc-nic.json ./virtCCA_sdk/kata-v3.15.0/conf/pcipc-nvme.json /etc/cdi +``` +/etc/cdi下网卡和磁盘设备配置文件中用户需要关注并针对性修改的是: +- name:该设备的唯一标志,不同设备name彼此不同,容器配置中通过指定name来注入对应设备。 +- path:该设备对应的vfio路径,参考上文**创建vfio设备**小节创建该路径。 +(devices数组支持添加多个设备的描述) + +4. k8s启动机密容器并直通网卡 +步骤: +- 1)完成vfio设备创建。 +- 2)修改`/etc/cdi/pcipc-nic.json`完成待直通的vfio设备配置(name和path)。 +- 3)容器配置文件.yaml中新增`cdi.k8s.io/vfio-pcipc`注解和initContainer预配网(可选),示例配置如下: +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: test-kata-qemu-virtcca + annotations: + io.containerd.cri.runtime-handler: "kata-qemu-virtcca" + cdi.k8s.io/vfio-pcipc: "pcipc/nic=1" # 1即/etc/cdi/pcipc-nic.json中的name字段内容 +spec: + runtimeClassName: kata-qemu-virtcca + terminationGracePeriodSeconds: 5 + initContainers: # 用于使用直通的网卡预配置网络(可选) + - name: network-setup + image: registry.hw.com:5000/ubuntu-net:latest + imagePullPolicy: Always + securityContext: + privileged: true + command: ["sh", "-c"] + args: ["ip link set eth1 up && ip addr add 192.168.100.90/24 dev eth1 && ip route replace default via 192.168.100.1"] + containers: + - name: box-1 + image: registry.hw.com:5000/busybox:latest + imagePullPolicy: Always + command: + - sh + tty: true +``` +- 4)启动机密容器,进到容器中ip a可以看到直通的网卡。 + +5. k8s启动机密容器并直通nvme磁盘 +步骤: +- 1)完成vfio设备创建。 +- 2)修改`/etc/cdi/pcipc-nvme.json`完成待直通的vfio设备配置(name和path)。 +- 3)修改`kata-qemu-virtcca`容器运行时配置文件打开guest_hook_path注释: + `vim /opt/kata/share/defaults/kata-containers/configuration-qemu-virtcca.toml`,确保`hypervisor.qemu`标签下:`guest_hook_path = "/usr/share/oci/hooks"`。 +- 4)部署磁盘挂载hook脚本到文件系统: +```shell +# 确定待直通的nvme磁盘名(需支持SRIOV),针对性修改`./virtCCA_sdk/kata-v3.15.0/conf/pcipc-nvme-hook.sh`中的`$DEVICE`宏定义值 +mount -o loop,offset=3145728 /opt/kata/share/kata-containers/kata-containers-confidential.img /mnt +mkdir -p /mnt/usr/share/oci/hooks/prestart +cd virtCCA_sdk && cp ./kata-v3.15.0/conf/pcipc-nvme-hook.sh /mnt/usr/share/oci/hooks/prestart +umount /mnt +``` +- 5)容器配置文件.yaml中新增`cdi.k8s.io/vfio-pcipc`注解,示例配置如下: +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: test-kata-qemu-virtcca + annotations: + io.containerd.cri.runtime-handler: "kata-qemu-virtcca" + cdi.k8s.io/vfio-pcipc: "pcipc/nvme=1" # 1即/etc/cdi/pcipc-nvme.json中的name字段内容 +spec: + runtimeClassName: kata-qemu-virtcca + terminationGracePeriodSeconds: 5 + containers: + - name: box-1 + image: registry.hw.com:5000/busybox:latest + imagePullPolicy: Always + command: + - sh + tty: true +``` +6)启动机密容器,容器根目录下新增`pcipci_disk`目录,即直通磁盘的挂载点,可直接读写。 \ No newline at end of file