From 7b82bdae1d3c6bca0829bf4df575b5f9d0b9bd64 Mon Sep 17 00:00:00 2001 From: ikarosYuuki Date: Fri, 9 Aug 2024 15:49:36 +0800 Subject: [PATCH] kata patch --- .../0001-adapt-for-cvm.patch | 339 ++++++++++++++++++ .../0002-support-virtiofs.patch | 60 ++++ 2 files changed, 399 insertions(+) create mode 100644 confidential_container/0001-adapt-for-cvm.patch create mode 100644 confidential_container/0002-support-virtiofs.patch diff --git a/confidential_container/0001-adapt-for-cvm.patch b/confidential_container/0001-adapt-for-cvm.patch new file mode 100644 index 0000000..20151b9 --- /dev/null +++ b/confidential_container/0001-adapt-for-cvm.patch @@ -0,0 +1,339 @@ +From 214f705a1a5acb148f0e3746032f8a25bf132068 Mon Sep 17 00:00:00 2001 +From: XiaoFeng Ma +Date: Mon, 12 Aug 2024 17:16:16 +0800 +Subject: [PATCH] adapt for cvm + +--- + src/agent/Makefile | 2 +- + src/agent/rustjail/src/mount.rs | 3 + + src/runtime/pkg/govmm/qemu/qemu.go | 84 ++++++++++-------------- + src/runtime/pkg/katautils/create.go | 6 +- + src/runtime/virtcontainers/hypervisor.go | 2 +- + src/runtime/virtcontainers/qemu.go | 1 - + src/runtime/virtcontainers/qemu_arm64.go | 10 ++- + 7 files changed, 49 insertions(+), 59 deletions(-) + +diff --git a/src/agent/Makefile b/src/agent/Makefile +index a3eb56705..969548f9a 100644 +--- a/src/agent/Makefile ++++ b/src/agent/Makefile +@@ -26,7 +26,7 @@ export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) + EXTRA_RUSTFEATURES := + + ##VAR SECCOMP=yes|no define if agent enables seccomp feature +-SECCOMP ?= yes ++SECCOMP ?= no + + # Enable seccomp feature of rust build + ifeq ($(SECCOMP),yes) +diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs +index b822736dc..3c16f0fb9 100644 +--- a/src/agent/rustjail/src/mount.rs ++++ b/src/agent/rustjail/src/mount.rs +@@ -219,6 +219,9 @@ pub fn init_rootfs( + } + + if m.r#type == "cgroup" { ++ continue; ++ } ++ if m.r#type == "cgroup_nouse" { + mount_cgroups(cfd_log, m, rootfs, flags, &data, cpath, mounts)?; + } else { + if m.destination == "/dev" { +diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go +index 5b618eb01..1470a3533 100644 +--- a/src/runtime/pkg/govmm/qemu/qemu.go ++++ b/src/runtime/pkg/govmm/qemu/qemu.go +@@ -584,6 +584,8 @@ func (fsdev FSDevice) QemuParams(config *Config) []string { + fsParams = append(fsParams, fmt.Sprintf("multidevs=%s", fsdev.Multidev)) + } + ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + +@@ -1057,6 +1059,7 @@ func (netdev NetDevice) QemuParams(config *Config) []string { + + if netdev.Type.QemuDeviceParam(&netdev, config) != "" { + deviceParams = netdev.QemuDeviceParams(config) ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") + if deviceParams != nil { + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) +@@ -1142,7 +1145,8 @@ func (dev SerialDevice) QemuParams(config *Config) []string { + var deviceParams []string + var qemuParams []string + +- deviceParams = append(deviceParams, dev.deviceName(config)) ++ var devName = dev.deviceName(config) ++ deviceParams = append(deviceParams, devName) + if s := dev.Transport.disableModern(config, dev.DisableModern); s != "" { + deviceParams = append(deviceParams, s) + } +@@ -1162,6 +1166,9 @@ func (dev SerialDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if devName == "virtio-serial-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -1307,6 +1314,7 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + qemuParams = append(qemuParams, "-drive") +@@ -1724,8 +1732,6 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", b.ROMFile)) + } + +- qemuParams = append(qemuParams, "-device") +- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + return qemuParams + } + +@@ -2007,6 +2013,9 @@ func (scsiCon SCSIController) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if driver == "virtio-scsi-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2125,9 +2134,6 @@ func (bridgeDev BridgeDevice) QemuParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("pref64-reserve=%s", bridgeDev.Pref64Reserve)) + } + +- qemuParams = append(qemuParams, "-device") +- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) +- + return qemuParams + } + +@@ -2213,6 +2219,9 @@ func (vsock VSOCKDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if driver == "vhost-vsock-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2272,7 +2281,8 @@ func (v RngDevice) QemuParams(config *Config) []string { + objectParams = append(objectParams, "rng-random") + objectParams = append(objectParams, "id="+v.ID) + +- deviceParams = append(deviceParams, v.deviceName(config)) ++ var devName = v.deviceName(config) ++ deviceParams = append(deviceParams, devName) + deviceParams = append(deviceParams, "rng="+v.ID) + + if v.Transport.isVirtioPCI(config) && v.ROMFile != "" { +@@ -2302,6 +2312,9 @@ func (v RngDevice) QemuParams(config *Config) []string { + qemuParams = append(qemuParams, strings.Join(objectParams, ",")) + + qemuParams = append(qemuParams, "-device") ++ if devName == "virtio-rng-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2854,6 +2867,8 @@ func (config *Config) appendMachine() { + machineParams = append(machineParams, config.Machine.Options) + } + ++ machineParams = append(machineParams, "kvm-type=cvm") ++ + config.qemuParams = append(config.qemuParams, "-machine") + config.qemuParams = append(config.qemuParams, strings.Join(machineParams, ",")) + } +@@ -2862,7 +2877,7 @@ func (config *Config) appendMachine() { + func (config *Config) appendCPUModel() { + if config.CPUModel != "" { + config.qemuParams = append(config.qemuParams, "-cpu") +- config.qemuParams = append(config.qemuParams, config.CPUModel) ++ config.qemuParams = append(config.qemuParams, "host,kvm-steal-time=off,kvm-no-adjvtime=on") + } + } + +@@ -2923,14 +2938,6 @@ func (config *Config) appendMemory() { + + memoryParams = append(memoryParams, config.Memory.Size) + +- if config.Memory.Slots > 0 { +- memoryParams = append(memoryParams, fmt.Sprintf("slots=%d", config.Memory.Slots)) +- } +- +- if config.Memory.MaxMem != "" { +- memoryParams = append(memoryParams, fmt.Sprintf("maxmem=%s", config.Memory.MaxMem)) +- } +- + config.qemuParams = append(config.qemuParams, "-m") + config.qemuParams = append(config.qemuParams, strings.Join(memoryParams, ",")) + } +@@ -2942,26 +2949,6 @@ func (config *Config) appendCPUs() error { + + SMPParams = append(SMPParams, fmt.Sprintf("%d", config.SMP.CPUs)) + +- if config.SMP.Cores > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("cores=%d", config.SMP.Cores)) +- } +- +- if config.SMP.Threads > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("threads=%d", config.SMP.Threads)) +- } +- +- if config.SMP.Sockets > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("sockets=%d", config.SMP.Sockets)) +- } +- +- if config.SMP.MaxCPUs > 0 { +- if config.SMP.MaxCPUs < config.SMP.CPUs { +- return fmt.Errorf("MaxCPUs %d must be equal to or greater than CPUs %d", +- config.SMP.MaxCPUs, config.SMP.CPUs) +- } +- SMPParams = append(SMPParams, fmt.Sprintf("maxcpus=%d", config.SMP.MaxCPUs)) +- } +- + config.qemuParams = append(config.qemuParams, "-smp") + config.qemuParams = append(config.qemuParams, strings.Join(SMPParams, ",")) + } +@@ -3054,13 +3041,9 @@ func (config *Config) appendMemoryKnobs() { + config.qemuParams = append(config.qemuParams, "-object") + config.qemuParams = append(config.qemuParams, objMemParam) + +- if isDimmSupported(config) { +- config.qemuParams = append(config.qemuParams, "-numa") +- config.qemuParams = append(config.qemuParams, numaMemParam) +- } else { +- config.qemuParams = append(config.qemuParams, "-machine") +- config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) +- } ++ numaMemParam = numaMemParam ++ config.qemuParams = append(config.qemuParams, "-machine") ++ config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) + } + + func (config *Config) appendKnobs() { +@@ -3068,9 +3051,7 @@ func (config *Config) appendKnobs() { + config.qemuParams = append(config.qemuParams, "-no-user-config") + } + +- if config.Knobs.NoDefaults { +- config.qemuParams = append(config.qemuParams, "-nodefaults") +- } ++ config.qemuParams = append(config.qemuParams, "--enable-kvm") + + if config.Knobs.NoGraphic { + config.qemuParams = append(config.qemuParams, "-nographic") +@@ -3154,6 +3135,11 @@ func (config *Config) appendFwCfg(logger QMPLog) { + } + } + ++func (config *Config) appendTMMGUEST() { ++ config.qemuParams = append(config.qemuParams, "-object") ++ config.qemuParams = append(config.qemuParams, "tmm-guest,id=tmm0,sve-vector-length=128,num-pmu-counters=1") ++} ++ + // LaunchQemu can be used to launch a new qemu instance. + // + // The Config parameter contains a set of qemu parameters and settings. +@@ -3167,10 +3153,6 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) + config.appendQMPSockets() + config.appendMemory() + config.appendDevices(logger) +- config.appendRTC() +- config.appendGlobalParam() +- config.appendPFlashParam() +- config.appendVGA() + config.appendKnobs() + config.appendKernel() + config.appendBios() +@@ -3179,6 +3161,8 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) + config.appendPidFile() + config.appendFwCfg(logger) + config.appendSeccompSandbox() ++ // append TMM GUEST ++ config.appendTMMGUEST() + + if err := config.appendCPUs(); err != nil { + return nil, nil, err +diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go +index bcc7d8ce7..07238aa03 100644 +--- a/src/runtime/pkg/katautils/create.go ++++ b/src/runtime/pkg/katautils/create.go +@@ -38,11 +38,7 @@ var systemdKernelParam = []vc.Param{ + }, + { + Key: "systemd.mask", +- Value: "systemd-networkd.service", +- }, +- { +- Key: "systemd.mask", +- Value: "systemd-networkd.socket", ++ Value: "NetworkManager.service", + }, + } + +diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go +index c61b44f67..5c726515d 100644 +--- a/src/runtime/virtcontainers/hypervisor.go ++++ b/src/runtime/virtcontainers/hypervisor.go +@@ -99,7 +99,7 @@ type RootfsDriver string + + const ( + // VirtioBlk is the Virtio-Blk rootfs driver. +- VirtioBlk RootfsDriver = "/dev/vda1" ++ VirtioBlk RootfsDriver = "/dev/vda" + + // Nvdimm is the Nvdimm rootfs driver. + Nvdimm RootfsType = "/dev/pmem0p1" +diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go +index 27d75ecc4..660a78dc5 100644 +--- a/src/runtime/virtcontainers/qemu.go ++++ b/src/runtime/virtcontainers/qemu.go +@@ -406,7 +406,6 @@ func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu + + // Add bridges before any other devices. This way we make sure that + // bridge gets the first available PCI address i.e bridgePCIStartAddr +- devices = q.arch.appendBridges(devices) + + devices, err = q.arch.appendConsole(ctx, devices, console) + if err != nil { +diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go +index 9e05c5452..814a27d51 100644 +--- a/src/runtime/virtcontainers/qemu_arm64.go ++++ b/src/runtime/virtcontainers/qemu_arm64.go +@@ -29,10 +29,18 @@ const defaultQemuMachineType = QemuVirt + + const qmpMigrationWaitTimeout = 10 * time.Second + +-const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host" ++// add cvm qemu machine options ++const defaultQemuMachineOptions = "gic-version=3,accel=kvm,kernel_irqchip=on" + + var kernelParams = []Param{ + {"iommu.passthrough", "0"}, ++ {"swiotlb", "262144,force"}, ++ {"console", "tty0"}, ++ {"console", "ttyAMA0"}, ++ {"kaslr.disabled", "1"}, ++ {"rodata", "off"}, ++ {"cma", "64M"}, ++ {"cvm_guest", "1"}, + } + + var supportedQemuMachine = govmmQemu.Machine{ +-- +2.27.0 + diff --git a/confidential_container/0002-support-virtiofs.patch b/confidential_container/0002-support-virtiofs.patch new file mode 100644 index 0000000..997b198 --- /dev/null +++ b/confidential_container/0002-support-virtiofs.patch @@ -0,0 +1,60 @@ +From 7a83166689f8e0cfc703d2dbca7baf4fd3753012 Mon Sep 17 00:00:00 2001 +From: XiaoFeng Ma +Date: Mon, 12 Aug 2024 17:30:35 +0800 +Subject: [PATCH] support virtiofs + +--- + src/runtime/pkg/govmm/qemu/qemu.go | 12 ++++++++++++ + src/runtime/virtcontainers/virtiofsd.go | 9 +++++++-- + 2 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go +index 1470a3533..812af4609 100644 +--- a/src/runtime/pkg/govmm/qemu/qemu.go ++++ b/src/runtime/pkg/govmm/qemu/qemu.go +@@ -1589,9 +1589,21 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile)) + } + ++ if driver == "vhost-user-fs-pci" { ++ deviceParams = append(deviceParams, "iommu_platform=on") ++ } ++ + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + ++ if driver == "vhost-user-fs-pci" { ++ qemuParams = append(qemuParams, "-object") ++ objParams := "memory-backend-file,id=mem,size=" + config.Memory.Size + ",mem-path=/dev/shm,share=on" ++ qemuParams = append(qemuParams, objParams) ++ qemuParams = append(qemuParams, "-numa") ++ qemuParams = append(qemuParams, "node,memdev=mem") ++ } ++ + return qemuParams + } + +diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go +index 3e02756eb..ff46691bd 100644 +--- a/src/runtime/virtcontainers/virtiofsd.go ++++ b/src/runtime/virtcontainers/virtiofsd.go +@@ -186,9 +186,14 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { + // Send logs to syslog + "--syslog", + // cache mode for virtiofsd +- "--cache=" + v.cache, ++ "-o", ++ "cache=" + v.cache, + // shared directory tree +- "--shared-dir=" + v.sourcePath, ++ "-o", ++ "source=" + v.sourcePath, ++ // Announce sub-mount points to the guest ++ "-o", ++ "announce_submounts", + // fd number of vhost-user socket + fmt.Sprintf("--fd=%v", FdSocketNumber), + } +-- +2.33.0 + -- Gitee