From 101744badc3ae3d42e7ae9712466a1c98f338135 Mon Sep 17 00:00:00 2001 From: ikarosYuuki Date: Fri, 9 Aug 2024 15:49:36 +0800 Subject: [PATCH] kata patch --- .../0001-adapt-for-cvm.patch | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 confidential_container/0001-adapt-for-cvm.patch diff --git a/confidential_container/0001-adapt-for-cvm.patch b/confidential_container/0001-adapt-for-cvm.patch new file mode 100644 index 0000000..917c229 --- /dev/null +++ b/confidential_container/0001-adapt-for-cvm.patch @@ -0,0 +1,353 @@ +From f968351f290bc41e70893361a0effec22288cf64 Mon Sep 17 00:00:00 2001 +From: XiaoFeng Ma +Date: Fri, 9 Aug 2024 15:35:09 +0800 +Subject: [PATCH] adapt for cvm + +--- + src/agent/Makefile | 2 +- + src/agent/rustjail/src/mount.rs | 4 ++ + src/runtime/pkg/govmm/qemu/qemu.go | 85 +++++++++--------------- + src/runtime/pkg/katautils/create.go | 6 +- + src/runtime/virtcontainers/hypervisor.go | 2 +- + src/runtime/virtcontainers/qemu.go | 4 -- + src/runtime/virtcontainers/qemu_arm64.go | 9 ++- + 7 files changed, 48 insertions(+), 64 deletions(-) + +diff --git a/src/agent/Makefile b/src/agent/Makefile +index a3eb56705..969548f9a 100644 +--- a/src/agent/Makefile ++++ b/src/agent/Makefile +@@ -26,7 +26,7 @@ export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) + EXTRA_RUSTFEATURES := + + ##VAR SECCOMP=yes|no define if agent enables seccomp feature +-SECCOMP ?= yes ++SECCOMP ?= no + + # Enable seccomp feature of rust build + ifeq ($(SECCOMP),yes) +diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs +index b822736dc..32626b5a8 100644 +--- a/src/agent/rustjail/src/mount.rs ++++ b/src/agent/rustjail/src/mount.rs +@@ -219,6 +219,10 @@ pub fn init_rootfs( + } + + if m.r#type == "cgroup" { ++ continue; ++ } ++ ++ if m.r#type == "cgroup_nouse" { + mount_cgroups(cfd_log, m, rootfs, flags, &data, cpath, mounts)?; + } else { + if m.destination == "/dev" { +diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go +index 5b618eb01..d67ab7791 100644 +--- a/src/runtime/pkg/govmm/qemu/qemu.go ++++ b/src/runtime/pkg/govmm/qemu/qemu.go +@@ -584,6 +584,8 @@ func (fsdev FSDevice) QemuParams(config *Config) []string { + fsParams = append(fsParams, fmt.Sprintf("multidevs=%s", fsdev.Multidev)) + } + ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + +@@ -1049,6 +1051,7 @@ func (netdev NetDevice) QemuParams(config *Config) []string { + + if netdev.Type.QemuNetdevParam(&netdev, config) != "" { + netdevParams = netdev.QemuNetdevParams(config) ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") + if netdevParams != nil { + qemuParams = append(qemuParams, "-netdev") + qemuParams = append(qemuParams, strings.Join(netdevParams, ",")) +@@ -1142,7 +1145,8 @@ func (dev SerialDevice) QemuParams(config *Config) []string { + var deviceParams []string + var qemuParams []string + +- deviceParams = append(deviceParams, dev.deviceName(config)) ++ var devName = dev.deviceName(config) ++ deviceParams = append(deviceParams, devName) + if s := dev.Transport.disableModern(config, dev.DisableModern); s != "" { + deviceParams = append(deviceParams, s) + } +@@ -1162,6 +1166,9 @@ func (dev SerialDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if devName == "virtio-serial-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -1307,6 +1314,7 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + qemuParams = append(qemuParams, "-drive") +@@ -1724,8 +1732,6 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", b.ROMFile)) + } + +- qemuParams = append(qemuParams, "-device") +- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + return qemuParams + } + +@@ -2007,6 +2013,9 @@ func (scsiCon SCSIController) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if driver == "virtio-scsi-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2125,9 +2134,6 @@ func (bridgeDev BridgeDevice) QemuParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("pref64-reserve=%s", bridgeDev.Pref64Reserve)) + } + +- qemuParams = append(qemuParams, "-device") +- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) +- + return qemuParams + } + +@@ -2213,6 +2219,9 @@ func (vsock VSOCKDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if driver == "vhost-vsock-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2272,7 +2281,8 @@ func (v RngDevice) QemuParams(config *Config) []string { + objectParams = append(objectParams, "rng-random") + objectParams = append(objectParams, "id="+v.ID) + +- deviceParams = append(deviceParams, v.deviceName(config)) ++ var devName = v.deviceName(config) ++ deviceParams = append(deviceParams, devName) + deviceParams = append(deviceParams, "rng="+v.ID) + + if v.Transport.isVirtioPCI(config) && v.ROMFile != "" { +@@ -2302,6 +2312,9 @@ func (v RngDevice) QemuParams(config *Config) []string { + qemuParams = append(qemuParams, strings.Join(objectParams, ",")) + + qemuParams = append(qemuParams, "-device") ++ if devName == "virtio-rng-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2854,6 +2867,8 @@ func (config *Config) appendMachine() { + machineParams = append(machineParams, config.Machine.Options) + } + ++ machineParams = append(machineParams, "kvm-type=cvm") ++ + config.qemuParams = append(config.qemuParams, "-machine") + config.qemuParams = append(config.qemuParams, strings.Join(machineParams, ",")) + } +@@ -2862,7 +2877,7 @@ func (config *Config) appendMachine() { + func (config *Config) appendCPUModel() { + if config.CPUModel != "" { + config.qemuParams = append(config.qemuParams, "-cpu") +- config.qemuParams = append(config.qemuParams, config.CPUModel) ++ config.qemuParams = append(config.qemuParams, "host,kvm-steal-time=off,kvm-no-adjvtime=on") + } + } + +@@ -2922,15 +2937,6 @@ func (config *Config) appendMemory() { + var memoryParams []string + + memoryParams = append(memoryParams, config.Memory.Size) +- +- if config.Memory.Slots > 0 { +- memoryParams = append(memoryParams, fmt.Sprintf("slots=%d", config.Memory.Slots)) +- } +- +- if config.Memory.MaxMem != "" { +- memoryParams = append(memoryParams, fmt.Sprintf("maxmem=%s", config.Memory.MaxMem)) +- } +- + config.qemuParams = append(config.qemuParams, "-m") + config.qemuParams = append(config.qemuParams, strings.Join(memoryParams, ",")) + } +@@ -2942,30 +2948,10 @@ func (config *Config) appendCPUs() error { + + SMPParams = append(SMPParams, fmt.Sprintf("%d", config.SMP.CPUs)) + +- if config.SMP.Cores > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("cores=%d", config.SMP.Cores)) +- } +- +- if config.SMP.Threads > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("threads=%d", config.SMP.Threads)) +- } +- +- if config.SMP.Sockets > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("sockets=%d", config.SMP.Sockets)) +- } +- +- if config.SMP.MaxCPUs > 0 { +- if config.SMP.MaxCPUs < config.SMP.CPUs { +- return fmt.Errorf("MaxCPUs %d must be equal to or greater than CPUs %d", +- config.SMP.MaxCPUs, config.SMP.CPUs) +- } +- SMPParams = append(SMPParams, fmt.Sprintf("maxcpus=%d", config.SMP.MaxCPUs)) +- } +- + config.qemuParams = append(config.qemuParams, "-smp") + config.qemuParams = append(config.qemuParams, strings.Join(SMPParams, ",")) +- } + ++ } + return nil + } + +@@ -3054,13 +3040,9 @@ func (config *Config) appendMemoryKnobs() { + config.qemuParams = append(config.qemuParams, "-object") + config.qemuParams = append(config.qemuParams, objMemParam) + +- if isDimmSupported(config) { +- config.qemuParams = append(config.qemuParams, "-numa") +- config.qemuParams = append(config.qemuParams, numaMemParam) +- } else { +- config.qemuParams = append(config.qemuParams, "-machine") +- config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) +- } ++ numaMemParam = numaMemParam ++ config.qemuParams = append(config.qemuParams, "-machine") ++ config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) + } + + func (config *Config) appendKnobs() { +@@ -3068,9 +3050,7 @@ func (config *Config) appendKnobs() { + config.qemuParams = append(config.qemuParams, "-no-user-config") + } + +- if config.Knobs.NoDefaults { +- config.qemuParams = append(config.qemuParams, "-nodefaults") +- } ++ config.qemuParams = append(config.qemuParams, "--enable-kvm") + + if config.Knobs.NoGraphic { + config.qemuParams = append(config.qemuParams, "-nographic") +@@ -3088,8 +3068,6 @@ func (config *Config) appendKnobs() { + config.qemuParams = append(config.qemuParams, "-daemonize") + } + +- config.appendMemoryKnobs() +- + if config.Knobs.Mlock { + config.qemuParams = append(config.qemuParams, "-overcommit") + config.qemuParams = append(config.qemuParams, "mem-lock=on") +@@ -3154,6 +3132,10 @@ func (config *Config) appendFwCfg(logger QMPLog) { + } + } + ++func (config *Config) appendTMMGUEST() { ++ config.qemuParams = append(config.qemuParams, "-object") ++ config.qemuParams = append(config.qemuParams, "tmm-guest,id=tmm0,sve-vector-length=128,num-pmu-counters=1") ++} + // LaunchQemu can be used to launch a new qemu instance. + // + // The Config parameter contains a set of qemu parameters and settings. +@@ -3170,7 +3152,6 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) + config.appendRTC() + config.appendGlobalParam() + config.appendPFlashParam() +- config.appendVGA() + config.appendKnobs() + config.appendKernel() + config.appendBios() +@@ -3179,7 +3160,7 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) + config.appendPidFile() + config.appendFwCfg(logger) + config.appendSeccompSandbox() +- ++ config.appendTMMGUEST() + if err := config.appendCPUs(); err != nil { + return nil, nil, err + } +diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go +index bcc7d8ce7..07238aa03 100644 +--- a/src/runtime/pkg/katautils/create.go ++++ b/src/runtime/pkg/katautils/create.go +@@ -38,11 +38,7 @@ var systemdKernelParam = []vc.Param{ + }, + { + Key: "systemd.mask", +- Value: "systemd-networkd.service", +- }, +- { +- Key: "systemd.mask", +- Value: "systemd-networkd.socket", ++ Value: "NetworkManager.service", + }, + } + +diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go +index c61b44f67..5c726515d 100644 +--- a/src/runtime/virtcontainers/hypervisor.go ++++ b/src/runtime/virtcontainers/hypervisor.go +@@ -99,7 +99,7 @@ type RootfsDriver string + + const ( + // VirtioBlk is the Virtio-Blk rootfs driver. +- VirtioBlk RootfsDriver = "/dev/vda1" ++ VirtioBlk RootfsDriver = "/dev/vda" + + // Nvdimm is the Nvdimm rootfs driver. + Nvdimm RootfsType = "/dev/pmem0p1" +diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go +index 27d75ecc4..7d927f698 100644 +--- a/src/runtime/virtcontainers/qemu.go ++++ b/src/runtime/virtcontainers/qemu.go +@@ -404,10 +404,6 @@ func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu + return nil, nil, err + } + +- // Add bridges before any other devices. This way we make sure that +- // bridge gets the first available PCI address i.e bridgePCIStartAddr +- devices = q.arch.appendBridges(devices) +- + devices, err = q.arch.appendConsole(ctx, devices, console) + if err != nil { + return nil, nil, err +diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go +index 9e05c5452..ebb6fcf94 100644 +--- a/src/runtime/virtcontainers/qemu_arm64.go ++++ b/src/runtime/virtcontainers/qemu_arm64.go +@@ -29,10 +29,17 @@ const defaultQemuMachineType = QemuVirt + + const qmpMigrationWaitTimeout = 10 * time.Second + +-const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host" ++const defaultQemuMachineOptions = "gic-version=3,accel=kvm,kernel_irq_chip=on" + + var kernelParams = []Param{ + {"iommu.passthrough", "0"}, ++ {"swiotlb", "262144,force"}, ++ {"console", "tty0"}, ++ {"console", "ttyAMA0"}, ++ {"kaslr.disabled", "1"}, ++ {"rodata", "off"}, ++ {"cma", "64M"}, ++ {"cvm_guest", "1"}, + } + + var supportedQemuMachine = govmmQemu.Machine{ +-- +2.27.0 + -- Gitee