From cbed19cf3dabb4de8d3b281eeeb0b8ff99a4e6e5 Mon Sep 17 00:00:00 2001 From: restricted-band-tree Date: Wed, 25 Sep 2024 09:58:30 +0800 Subject: [PATCH 1/3] change link to use local image-rs --- .../0001-adapt-for-cvm.patch | 24 +++++++++++++++---- ...iofs.patch => 0002-support-virtiofs.patch} | 8 +++---- 2 files changed, 23 insertions(+), 9 deletions(-) rename confidential_container/{0001-support-virtiofs.patch => 0002-support-virtiofs.patch} (92%) diff --git a/confidential_container/0001-adapt-for-cvm.patch b/confidential_container/0001-adapt-for-cvm.patch index 20151b9..97c6dc6 100644 --- a/confidential_container/0001-adapt-for-cvm.patch +++ b/confidential_container/0001-adapt-for-cvm.patch @@ -1,9 +1,10 @@ -From 214f705a1a5acb148f0e3746032f8a25bf132068 Mon Sep 17 00:00:00 2001 -From: XiaoFeng Ma -Date: Mon, 12 Aug 2024 17:16:16 +0800 +From ff216cfd7dd814e66d8facd0d9ea2c1ddf36bc08 Mon Sep 17 00:00:00 2001 +From: yxk +Date: Fri, 6 Sep 2024 14:43:24 +0800 Subject: [PATCH] adapt for cvm --- + src/agent/Cargo.toml | 2 +- src/agent/Makefile | 2 +- src/agent/rustjail/src/mount.rs | 3 + src/runtime/pkg/govmm/qemu/qemu.go | 84 ++++++++++-------------- @@ -11,8 +12,21 @@ Subject: [PATCH] adapt for cvm src/runtime/virtcontainers/hypervisor.go | 2 +- src/runtime/virtcontainers/qemu.go | 1 - src/runtime/virtcontainers/qemu_arm64.go | 10 ++- - 7 files changed, 49 insertions(+), 59 deletions(-) + 8 files changed, 50 insertions(+), 60 deletions(-) +diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml +index ad57a770c..3f68360ba 100644 +--- a/src/agent/Cargo.toml ++++ b/src/agent/Cargo.toml +@@ -74,7 +74,7 @@ clap = { version = "3.0.1", features = ["derive"] } + openssl = { version = "0.10.38", features = ["vendored"] } + + # Image pull/decrypt +-image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.8.0", default-features = false, features = [ ++image-rs = { path = "../../../guest-components/image-rs", default-features = false, features = [ + "kata-cc-native-tls", + "verity", + "signature-simple-xrss", diff --git a/src/agent/Makefile b/src/agent/Makefile index a3eb56705..969548f9a 100644 --- a/src/agent/Makefile @@ -335,5 +349,5 @@ index 9e05c5452..814a27d51 100644 var supportedQemuMachine = govmmQemu.Machine{ -- -2.27.0 +2.33.0 diff --git a/confidential_container/0001-support-virtiofs.patch b/confidential_container/0002-support-virtiofs.patch similarity index 92% rename from confidential_container/0001-support-virtiofs.patch rename to confidential_container/0002-support-virtiofs.patch index 7f11f68..997b198 100644 --- a/confidential_container/0001-support-virtiofs.patch +++ b/confidential_container/0002-support-virtiofs.patch @@ -1,4 +1,4 @@ -From 37f45b783b9d7aa6d1ce6336bf2fb7392648a481 Mon Sep 17 00:00:00 2001 +From 7a83166689f8e0cfc703d2dbca7baf4fd3753012 Mon Sep 17 00:00:00 2001 From: XiaoFeng Ma Date: Mon, 12 Aug 2024 17:30:35 +0800 Subject: [PATCH] support virtiofs @@ -9,7 +9,7 @@ Subject: [PATCH] support virtiofs 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go -index 1470a3533..e2ae4e5c5 100644 +index 1470a3533..812af4609 100644 --- a/src/runtime/pkg/govmm/qemu/qemu.go +++ b/src/runtime/pkg/govmm/qemu/qemu.go @@ -1589,9 +1589,21 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string { @@ -25,7 +25,7 @@ index 1470a3533..e2ae4e5c5 100644 + if driver == "vhost-user-fs-pci" { + qemuParams = append(qemuParams, "-object") -+ objParams := "memory-backend-file,id=mem,size=" + config.Memory.Size + ", mem-path=/dev/shm,share=on" ++ objParams := "memory-backend-file,id=mem,size=" + config.Memory.Size + ",mem-path=/dev/shm,share=on" + qemuParams = append(qemuParams, objParams) + qemuParams = append(qemuParams, "-numa") + qemuParams = append(qemuParams, "node,memdev=mem") @@ -56,5 +56,5 @@ index 3e02756eb..ff46691bd 100644 fmt.Sprintf("--fd=%v", FdSocketNumber), } -- -2.27.0 +2.33.0 -- Gitee From 33a781d96b34343628da4f275661301daefe3efd Mon Sep 17 00:00:00 2001 From: restricted-band-tree Date: Wed, 25 Sep 2024 12:00:19 +0800 Subject: [PATCH 2/3] kata --- confidential_container/0001-kata.patch | 464 +++++++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 confidential_container/0001-kata.patch diff --git a/confidential_container/0001-kata.patch b/confidential_container/0001-kata.patch new file mode 100644 index 0000000..2822c58 --- /dev/null +++ b/confidential_container/0001-kata.patch @@ -0,0 +1,464 @@ +From 3b5320473bc543eceef163325dbda5d26e9239f3 Mon Sep 17 00:00:00 2001 +From: panhengchang +Date: Wed, 25 Sep 2024 11:31:02 +0800 +Subject: [PATCH] kata + +--- + src/agent/Cargo.lock | 2 - + src/agent/Cargo.toml | 2 +- + src/agent/Makefile | 2 +- + src/agent/rustjail/src/mount.rs | 3 + + src/runtime/pkg/govmm/qemu/qemu.go | 135 +++++++++++++++-------- + src/runtime/pkg/katautils/create.go | 6 +- + src/runtime/virtcontainers/hypervisor.go | 2 +- + src/runtime/virtcontainers/qemu.go | 1 - + src/runtime/virtcontainers/qemu_arm64.go | 10 +- + src/runtime/virtcontainers/virtiofsd.go | 9 +- + 10 files changed, 114 insertions(+), 58 deletions(-) + +diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock +index 4a87c6a2f..341f0e7b2 100644 +--- a/src/agent/Cargo.lock ++++ b/src/agent/Cargo.lock +@@ -2015,7 +2015,6 @@ dependencies = [ + [[package]] + name = "image-rs" + version = "0.1.0" +-source = "git+https://github.com/confidential-containers/guest-components?tag=v0.8.0#e849dc8921d2a48bec915f1a7c02f8988721022d" + dependencies = [ + "anyhow", + "async-compression", +@@ -2921,7 +2920,6 @@ dependencies = [ + [[package]] + name = "ocicrypt-rs" + version = "0.1.0" +-source = "git+https://github.com/confidential-containers/guest-components?tag=v0.8.0#e849dc8921d2a48bec915f1a7c02f8988721022d" + dependencies = [ + "aes 0.8.3", + "anyhow", +diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml +index ad57a770c..3f68360ba 100644 +--- a/src/agent/Cargo.toml ++++ b/src/agent/Cargo.toml +@@ -74,7 +74,7 @@ clap = { version = "3.0.1", features = ["derive"] } + openssl = { version = "0.10.38", features = ["vendored"] } + + # Image pull/decrypt +-image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.8.0", default-features = false, features = [ ++image-rs = { path = "../../../guest-components/image-rs", default-features = false, features = [ + "kata-cc-native-tls", + "verity", + "signature-simple-xrss", +diff --git a/src/agent/Makefile b/src/agent/Makefile +index a3eb56705..969548f9a 100644 +--- a/src/agent/Makefile ++++ b/src/agent/Makefile +@@ -26,7 +26,7 @@ export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) + EXTRA_RUSTFEATURES := + + ##VAR SECCOMP=yes|no define if agent enables seccomp feature +-SECCOMP ?= yes ++SECCOMP ?= no + + # Enable seccomp feature of rust build + ifeq ($(SECCOMP),yes) +diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs +index b822736dc..3c16f0fb9 100644 +--- a/src/agent/rustjail/src/mount.rs ++++ b/src/agent/rustjail/src/mount.rs +@@ -219,6 +219,9 @@ pub fn init_rootfs( + } + + if m.r#type == "cgroup" { ++ continue; ++ } ++ if m.r#type == "cgroup_nouse" { + mount_cgroups(cfd_log, m, rootfs, flags, &data, cpath, mounts)?; + } else { + if m.destination == "/dev" { +diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go +index 5b618eb01..a5a196c67 100644 +--- a/src/runtime/pkg/govmm/qemu/qemu.go ++++ b/src/runtime/pkg/govmm/qemu/qemu.go +@@ -584,6 +584,8 @@ func (fsdev FSDevice) QemuParams(config *Config) []string { + fsParams = append(fsParams, fmt.Sprintf("multidevs=%s", fsdev.Multidev)) + } + ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + +@@ -1057,6 +1059,7 @@ func (netdev NetDevice) QemuParams(config *Config) []string { + + if netdev.Type.QemuDeviceParam(&netdev, config) != "" { + deviceParams = netdev.QemuDeviceParams(config) ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") + if deviceParams != nil { + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) +@@ -1142,7 +1145,8 @@ func (dev SerialDevice) QemuParams(config *Config) []string { + var deviceParams []string + var qemuParams []string + +- deviceParams = append(deviceParams, dev.deviceName(config)) ++ var devName = dev.deviceName(config) ++ deviceParams = append(deviceParams, devName) + if s := dev.Transport.disableModern(config, dev.DisableModern); s != "" { + deviceParams = append(deviceParams, s) + } +@@ -1162,6 +1166,9 @@ func (dev SerialDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if devName == "virtio-serial-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -1307,6 +1314,7 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + qemuParams = append(qemuParams, "-drive") +@@ -1581,6 +1589,10 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile)) + } + ++ if driver == "vhost-user-fs-pci" { ++ deviceParams = append(deviceParams, "iommu_platform=on") ++ } ++ + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + +@@ -1724,8 +1736,6 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", b.ROMFile)) + } + +- qemuParams = append(qemuParams, "-device") +- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + return qemuParams + } + +@@ -2007,6 +2017,9 @@ func (scsiCon SCSIController) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if driver == "virtio-scsi-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2125,9 +2138,6 @@ func (bridgeDev BridgeDevice) QemuParams(config *Config) []string { + deviceParams = append(deviceParams, fmt.Sprintf("pref64-reserve=%s", bridgeDev.Pref64Reserve)) + } + +- qemuParams = append(qemuParams, "-device") +- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) +- + return qemuParams + } + +@@ -2213,6 +2223,9 @@ func (vsock VSOCKDevice) QemuParams(config *Config) []string { + } + + qemuParams = append(qemuParams, "-device") ++ if driver == "vhost-vsock-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2272,7 +2285,8 @@ func (v RngDevice) QemuParams(config *Config) []string { + objectParams = append(objectParams, "rng-random") + objectParams = append(objectParams, "id="+v.ID) + +- deviceParams = append(deviceParams, v.deviceName(config)) ++ var devName = v.deviceName(config) ++ deviceParams = append(deviceParams, devName) + deviceParams = append(deviceParams, "rng="+v.ID) + + if v.Transport.isVirtioPCI(config) && v.ROMFile != "" { +@@ -2302,6 +2316,9 @@ func (v RngDevice) QemuParams(config *Config) []string { + qemuParams = append(qemuParams, strings.Join(objectParams, ",")) + + qemuParams = append(qemuParams, "-device") ++ if devName == "virtio-rng-pci" { ++ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") ++ } + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + + return qemuParams +@@ -2854,6 +2871,9 @@ func (config *Config) appendMachine() { + machineParams = append(machineParams, config.Machine.Options) + } + ++ machineParams = append(machineParams, "kvm-type=cvm") ++ machineParams = append(machineParams, "kata-affinity=on") ++ + config.qemuParams = append(config.qemuParams, "-machine") + config.qemuParams = append(config.qemuParams, strings.Join(machineParams, ",")) + } +@@ -2862,7 +2882,7 @@ func (config *Config) appendMachine() { + func (config *Config) appendCPUModel() { + if config.CPUModel != "" { + config.qemuParams = append(config.qemuParams, "-cpu") +- config.qemuParams = append(config.qemuParams, config.CPUModel) ++ config.qemuParams = append(config.qemuParams, "host,kvm-steal-time=off,kvm-no-adjvtime=on") + } + } + +@@ -2923,44 +2943,21 @@ func (config *Config) appendMemory() { + + memoryParams = append(memoryParams, config.Memory.Size) + +- if config.Memory.Slots > 0 { +- memoryParams = append(memoryParams, fmt.Sprintf("slots=%d", config.Memory.Slots)) +- } +- +- if config.Memory.MaxMem != "" { +- memoryParams = append(memoryParams, fmt.Sprintf("maxmem=%s", config.Memory.MaxMem)) +- } +- + config.qemuParams = append(config.qemuParams, "-m") + config.qemuParams = append(config.qemuParams, strings.Join(memoryParams, ",")) + } + } + ++var ( ++ smp_cpu int ++) ++ + func (config *Config) appendCPUs() error { + if config.SMP.CPUs > 0 { + var SMPParams []string + + SMPParams = append(SMPParams, fmt.Sprintf("%d", config.SMP.CPUs)) +- +- if config.SMP.Cores > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("cores=%d", config.SMP.Cores)) +- } +- +- if config.SMP.Threads > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("threads=%d", config.SMP.Threads)) +- } +- +- if config.SMP.Sockets > 0 { +- SMPParams = append(SMPParams, fmt.Sprintf("sockets=%d", config.SMP.Sockets)) +- } +- +- if config.SMP.MaxCPUs > 0 { +- if config.SMP.MaxCPUs < config.SMP.CPUs { +- return fmt.Errorf("MaxCPUs %d must be equal to or greater than CPUs %d", +- config.SMP.MaxCPUs, config.SMP.CPUs) +- } +- SMPParams = append(SMPParams, fmt.Sprintf("maxcpus=%d", config.SMP.MaxCPUs)) +- } ++ smp_cpu = int(config.SMP.CPUs) + + config.qemuParams = append(config.qemuParams, "-smp") + config.qemuParams = append(config.qemuParams, strings.Join(SMPParams, ",")) +@@ -3068,9 +3065,7 @@ func (config *Config) appendKnobs() { + config.qemuParams = append(config.qemuParams, "-no-user-config") + } + +- if config.Knobs.NoDefaults { +- config.qemuParams = append(config.qemuParams, "-nodefaults") +- } ++ config.qemuParams = append(config.qemuParams, "--enable-kvm") + + if config.Knobs.NoGraphic { + config.qemuParams = append(config.qemuParams, "-nographic") +@@ -3154,6 +3149,11 @@ func (config *Config) appendFwCfg(logger QMPLog) { + } + } + ++func (config *Config) appendTMMGUEST() { ++ config.qemuParams = append(config.qemuParams, "-object") ++ config.qemuParams = append(config.qemuParams, "tmm-guest,id=tmm0,sve-vector-length=128,num-pmu-counters=1") ++} ++ + // LaunchQemu can be used to launch a new qemu instance. + // + // The Config parameter contains a set of qemu parameters and settings. +@@ -3167,10 +3167,6 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) + config.appendQMPSockets() + config.appendMemory() + config.appendDevices(logger) +- config.appendRTC() +- config.appendGlobalParam() +- config.appendPFlashParam() +- config.appendVGA() + config.appendKnobs() + config.appendKernel() + config.appendBios() +@@ -3179,6 +3175,8 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) + config.appendPidFile() + config.appendFwCfg(logger) + config.appendSeccompSandbox() ++ // append TMM GUEST ++ config.appendTMMGUEST() + + if err := config.appendCPUs(); err != nil { + return nil, nil, err +@@ -3226,8 +3224,52 @@ func LaunchCustomQemu(ctx context.Context, path string, params []string, fds []* + path = "qemu-system-x86_64" + } + ++ lock_file := "/root/kata_lock" ++ file, err := os.Create(lock_file) ++ if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX); err != nil { ++ logger.Errorf("add exclude lock in no block failed", err) ++ return nil, nil, err ++ } ++ ++ // get the existing CPU affinity list ++ pidCmd := exec.Command("pidof", "qemu-system-aarch64") ++ pidOutput, _ := pidCmd.Output() ++ pids := strings.Fields(string(pidOutput)) ++ ++ m := map[int]int{} ++ for _, pid := range pids { ++ tasksetCmd := exec.Command("taskset", "-pc", pid) ++ cpus, _ := tasksetCmd.Output() ++ num, _ := strconv.Atoi(strings.TrimSpace((strings.Split(strings.Split(string(cpus), ":")[1], "-")[0]))) ++ m[num] = -1 ++ } ++ ++ numCPU := runtime.NumCPU() ++ if smp_cpu > numCPU { ++ logger.Errorf("invalid config, smp %d is too large", smp_cpu) ++ return nil, nil, nil ++ } ++ ++ // compute CPU affinity for QEMU Command ++ exit := true ++ start_cpu := -smp_cpu ++ for exit { ++ start_cpu += smp_cpu ++ _, exit = m[start_cpu] ++ } ++ end_cpu := start_cpu + smp_cpu - 1 ++ if start_cpu >= numCPU || end_cpu >= numCPU { ++ start_cpu = 0 ++ end_cpu = start_cpu + smp_cpu - 1 ++ } ++ cpu := strconv.Itoa(start_cpu) + "-" + strconv.Itoa(end_cpu) ++ ++ // construct the QEMU Command ++ tasksetArgs := []string{"-c", cpu, path} ++ tasksetArgs = append(tasksetArgs, params...) ++ cmd := exec.CommandContext(ctx, "taskset", tasksetArgs...) ++ + /* #nosec */ +- cmd := exec.CommandContext(ctx, path, params...) + if len(fds) > 0 { + logger.Infof("Adding extra file %v", fds) + cmd.ExtraFiles = fds +@@ -3247,5 +3289,10 @@ func LaunchCustomQemu(ctx context.Context, path string, params []string, fds []* + logger.Errorf("Unable to launch %s: %v", path, err) + return nil, nil, err + } ++ ++ if err := syscall.Flock(int(file.Fd()), syscall.LOCK_UN); err != nil { ++ log.Println("unlock exclude lock failed", err) ++ } ++ + return cmd, reader, nil + } +diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go +index bcc7d8ce7..07238aa03 100644 +--- a/src/runtime/pkg/katautils/create.go ++++ b/src/runtime/pkg/katautils/create.go +@@ -38,11 +38,7 @@ var systemdKernelParam = []vc.Param{ + }, + { + Key: "systemd.mask", +- Value: "systemd-networkd.service", +- }, +- { +- Key: "systemd.mask", +- Value: "systemd-networkd.socket", ++ Value: "NetworkManager.service", + }, + } + +diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go +index c61b44f67..5c726515d 100644 +--- a/src/runtime/virtcontainers/hypervisor.go ++++ b/src/runtime/virtcontainers/hypervisor.go +@@ -99,7 +99,7 @@ type RootfsDriver string + + const ( + // VirtioBlk is the Virtio-Blk rootfs driver. +- VirtioBlk RootfsDriver = "/dev/vda1" ++ VirtioBlk RootfsDriver = "/dev/vda" + + // Nvdimm is the Nvdimm rootfs driver. + Nvdimm RootfsType = "/dev/pmem0p1" +diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go +index 27d75ecc4..660a78dc5 100644 +--- a/src/runtime/virtcontainers/qemu.go ++++ b/src/runtime/virtcontainers/qemu.go +@@ -406,7 +406,6 @@ func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu + + // Add bridges before any other devices. This way we make sure that + // bridge gets the first available PCI address i.e bridgePCIStartAddr +- devices = q.arch.appendBridges(devices) + + devices, err = q.arch.appendConsole(ctx, devices, console) + if err != nil { +diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go +index 9e05c5452..e984a3f2f 100644 +--- a/src/runtime/virtcontainers/qemu_arm64.go ++++ b/src/runtime/virtcontainers/qemu_arm64.go +@@ -29,10 +29,18 @@ const defaultQemuMachineType = QemuVirt + + const qmpMigrationWaitTimeout = 10 * time.Second + +-const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host" ++// add cvm qemu machine options ++const defaultQemuMachineOptions = "gic-version=3,accel=kvm,kernel_irqchip=on" + + var kernelParams = []Param{ + {"iommu.passthrough", "0"}, ++ {"swiotlb", "262144,force"}, ++ {"console", "tty0"}, ++ {"console", "ttyAMA0"}, ++ {"kaslr.disabled", "1"}, ++ {"rodata", "off"}, ++ {"cma", "64M"}, ++ {"virtcca_cvm_guest", "1"}, + } + + var supportedQemuMachine = govmmQemu.Machine{ +diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go +index 3e02756eb..ff46691bd 100644 +--- a/src/runtime/virtcontainers/virtiofsd.go ++++ b/src/runtime/virtcontainers/virtiofsd.go +@@ -186,9 +186,14 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { + // Send logs to syslog + "--syslog", + // cache mode for virtiofsd +- "--cache=" + v.cache, ++ "-o", ++ "cache=" + v.cache, + // shared directory tree +- "--shared-dir=" + v.sourcePath, ++ "-o", ++ "source=" + v.sourcePath, ++ // Announce sub-mount points to the guest ++ "-o", ++ "announce_submounts", + // fd number of vhost-user socket + fmt.Sprintf("--fd=%v", FdSocketNumber), + } +-- +2.33.0 + -- Gitee From 7f336904b6639728ed5d3a7a98eb92b67a139051 Mon Sep 17 00:00:00 2001 From: restricted-band-tree Date: Wed, 25 Sep 2024 16:40:34 +0800 Subject: [PATCH 3/3] kata patch for virtcca --- .../0001-adapt-for-cvm.patch | 353 ------------------ .../0002-support-virtiofs.patch | 60 --- .../{0001-kata.patch => kata-container.patch} | 0 3 files changed, 413 deletions(-) delete mode 100644 confidential_container/0001-adapt-for-cvm.patch delete mode 100644 confidential_container/0002-support-virtiofs.patch rename confidential_container/{0001-kata.patch => kata-container.patch} (100%) diff --git a/confidential_container/0001-adapt-for-cvm.patch b/confidential_container/0001-adapt-for-cvm.patch deleted file mode 100644 index 97c6dc6..0000000 --- a/confidential_container/0001-adapt-for-cvm.patch +++ /dev/null @@ -1,353 +0,0 @@ -From ff216cfd7dd814e66d8facd0d9ea2c1ddf36bc08 Mon Sep 17 00:00:00 2001 -From: yxk -Date: Fri, 6 Sep 2024 14:43:24 +0800 -Subject: [PATCH] adapt for cvm - ---- - src/agent/Cargo.toml | 2 +- - src/agent/Makefile | 2 +- - src/agent/rustjail/src/mount.rs | 3 + - src/runtime/pkg/govmm/qemu/qemu.go | 84 ++++++++++-------------- - src/runtime/pkg/katautils/create.go | 6 +- - src/runtime/virtcontainers/hypervisor.go | 2 +- - src/runtime/virtcontainers/qemu.go | 1 - - src/runtime/virtcontainers/qemu_arm64.go | 10 ++- - 8 files changed, 50 insertions(+), 60 deletions(-) - -diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml -index ad57a770c..3f68360ba 100644 ---- a/src/agent/Cargo.toml -+++ b/src/agent/Cargo.toml -@@ -74,7 +74,7 @@ clap = { version = "3.0.1", features = ["derive"] } - openssl = { version = "0.10.38", features = ["vendored"] } - - # Image pull/decrypt --image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.8.0", default-features = false, features = [ -+image-rs = { path = "../../../guest-components/image-rs", default-features = false, features = [ - "kata-cc-native-tls", - "verity", - "signature-simple-xrss", -diff --git a/src/agent/Makefile b/src/agent/Makefile -index a3eb56705..969548f9a 100644 ---- a/src/agent/Makefile -+++ b/src/agent/Makefile -@@ -26,7 +26,7 @@ export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) - EXTRA_RUSTFEATURES := - - ##VAR SECCOMP=yes|no define if agent enables seccomp feature --SECCOMP ?= yes -+SECCOMP ?= no - - # Enable seccomp feature of rust build - ifeq ($(SECCOMP),yes) -diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs -index b822736dc..3c16f0fb9 100644 ---- a/src/agent/rustjail/src/mount.rs -+++ b/src/agent/rustjail/src/mount.rs -@@ -219,6 +219,9 @@ pub fn init_rootfs( - } - - if m.r#type == "cgroup" { -+ continue; -+ } -+ if m.r#type == "cgroup_nouse" { - mount_cgroups(cfd_log, m, rootfs, flags, &data, cpath, mounts)?; - } else { - if m.destination == "/dev" { -diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go -index 5b618eb01..1470a3533 100644 ---- a/src/runtime/pkg/govmm/qemu/qemu.go -+++ b/src/runtime/pkg/govmm/qemu/qemu.go -@@ -584,6 +584,8 @@ func (fsdev FSDevice) QemuParams(config *Config) []string { - fsParams = append(fsParams, fmt.Sprintf("multidevs=%s", fsdev.Multidev)) - } - -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") -+ - qemuParams = append(qemuParams, "-device") - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - -@@ -1057,6 +1059,7 @@ func (netdev NetDevice) QemuParams(config *Config) []string { - - if netdev.Type.QemuDeviceParam(&netdev, config) != "" { - deviceParams = netdev.QemuDeviceParams(config) -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") - if deviceParams != nil { - qemuParams = append(qemuParams, "-device") - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) -@@ -1142,7 +1145,8 @@ func (dev SerialDevice) QemuParams(config *Config) []string { - var deviceParams []string - var qemuParams []string - -- deviceParams = append(deviceParams, dev.deviceName(config)) -+ var devName = dev.deviceName(config) -+ deviceParams = append(deviceParams, devName) - if s := dev.Transport.disableModern(config, dev.DisableModern); s != "" { - deviceParams = append(deviceParams, s) - } -@@ -1162,6 +1166,9 @@ func (dev SerialDevice) QemuParams(config *Config) []string { - } - - qemuParams = append(qemuParams, "-device") -+ if devName == "virtio-serial-pci" { -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") -+ } - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - - return qemuParams -@@ -1307,6 +1314,7 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string { - } - - qemuParams = append(qemuParams, "-device") -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - - qemuParams = append(qemuParams, "-drive") -@@ -1724,8 +1732,6 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { - deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", b.ROMFile)) - } - -- qemuParams = append(qemuParams, "-device") -- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - return qemuParams - } - -@@ -2007,6 +2013,9 @@ func (scsiCon SCSIController) QemuParams(config *Config) []string { - } - - qemuParams = append(qemuParams, "-device") -+ if driver == "virtio-scsi-pci" { -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") -+ } - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - - return qemuParams -@@ -2125,9 +2134,6 @@ func (bridgeDev BridgeDevice) QemuParams(config *Config) []string { - deviceParams = append(deviceParams, fmt.Sprintf("pref64-reserve=%s", bridgeDev.Pref64Reserve)) - } - -- qemuParams = append(qemuParams, "-device") -- qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) -- - return qemuParams - } - -@@ -2213,6 +2219,9 @@ func (vsock VSOCKDevice) QemuParams(config *Config) []string { - } - - qemuParams = append(qemuParams, "-device") -+ if driver == "vhost-vsock-pci" { -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") -+ } - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - - return qemuParams -@@ -2272,7 +2281,8 @@ func (v RngDevice) QemuParams(config *Config) []string { - objectParams = append(objectParams, "rng-random") - objectParams = append(objectParams, "id="+v.ID) - -- deviceParams = append(deviceParams, v.deviceName(config)) -+ var devName = v.deviceName(config) -+ deviceParams = append(deviceParams, devName) - deviceParams = append(deviceParams, "rng="+v.ID) - - if v.Transport.isVirtioPCI(config) && v.ROMFile != "" { -@@ -2302,6 +2312,9 @@ func (v RngDevice) QemuParams(config *Config) []string { - qemuParams = append(qemuParams, strings.Join(objectParams, ",")) - - qemuParams = append(qemuParams, "-device") -+ if devName == "virtio-rng-pci" { -+ deviceParams = append(deviceParams, "disable-legacy=on,iommu_platform=on") -+ } - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - - return qemuParams -@@ -2854,6 +2867,8 @@ func (config *Config) appendMachine() { - machineParams = append(machineParams, config.Machine.Options) - } - -+ machineParams = append(machineParams, "kvm-type=cvm") -+ - config.qemuParams = append(config.qemuParams, "-machine") - config.qemuParams = append(config.qemuParams, strings.Join(machineParams, ",")) - } -@@ -2862,7 +2877,7 @@ func (config *Config) appendMachine() { - func (config *Config) appendCPUModel() { - if config.CPUModel != "" { - config.qemuParams = append(config.qemuParams, "-cpu") -- config.qemuParams = append(config.qemuParams, config.CPUModel) -+ config.qemuParams = append(config.qemuParams, "host,kvm-steal-time=off,kvm-no-adjvtime=on") - } - } - -@@ -2923,14 +2938,6 @@ func (config *Config) appendMemory() { - - memoryParams = append(memoryParams, config.Memory.Size) - -- if config.Memory.Slots > 0 { -- memoryParams = append(memoryParams, fmt.Sprintf("slots=%d", config.Memory.Slots)) -- } -- -- if config.Memory.MaxMem != "" { -- memoryParams = append(memoryParams, fmt.Sprintf("maxmem=%s", config.Memory.MaxMem)) -- } -- - config.qemuParams = append(config.qemuParams, "-m") - config.qemuParams = append(config.qemuParams, strings.Join(memoryParams, ",")) - } -@@ -2942,26 +2949,6 @@ func (config *Config) appendCPUs() error { - - SMPParams = append(SMPParams, fmt.Sprintf("%d", config.SMP.CPUs)) - -- if config.SMP.Cores > 0 { -- SMPParams = append(SMPParams, fmt.Sprintf("cores=%d", config.SMP.Cores)) -- } -- -- if config.SMP.Threads > 0 { -- SMPParams = append(SMPParams, fmt.Sprintf("threads=%d", config.SMP.Threads)) -- } -- -- if config.SMP.Sockets > 0 { -- SMPParams = append(SMPParams, fmt.Sprintf("sockets=%d", config.SMP.Sockets)) -- } -- -- if config.SMP.MaxCPUs > 0 { -- if config.SMP.MaxCPUs < config.SMP.CPUs { -- return fmt.Errorf("MaxCPUs %d must be equal to or greater than CPUs %d", -- config.SMP.MaxCPUs, config.SMP.CPUs) -- } -- SMPParams = append(SMPParams, fmt.Sprintf("maxcpus=%d", config.SMP.MaxCPUs)) -- } -- - config.qemuParams = append(config.qemuParams, "-smp") - config.qemuParams = append(config.qemuParams, strings.Join(SMPParams, ",")) - } -@@ -3054,13 +3041,9 @@ func (config *Config) appendMemoryKnobs() { - config.qemuParams = append(config.qemuParams, "-object") - config.qemuParams = append(config.qemuParams, objMemParam) - -- if isDimmSupported(config) { -- config.qemuParams = append(config.qemuParams, "-numa") -- config.qemuParams = append(config.qemuParams, numaMemParam) -- } else { -- config.qemuParams = append(config.qemuParams, "-machine") -- config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) -- } -+ numaMemParam = numaMemParam -+ config.qemuParams = append(config.qemuParams, "-machine") -+ config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) - } - - func (config *Config) appendKnobs() { -@@ -3068,9 +3051,7 @@ func (config *Config) appendKnobs() { - config.qemuParams = append(config.qemuParams, "-no-user-config") - } - -- if config.Knobs.NoDefaults { -- config.qemuParams = append(config.qemuParams, "-nodefaults") -- } -+ config.qemuParams = append(config.qemuParams, "--enable-kvm") - - if config.Knobs.NoGraphic { - config.qemuParams = append(config.qemuParams, "-nographic") -@@ -3154,6 +3135,11 @@ func (config *Config) appendFwCfg(logger QMPLog) { - } - } - -+func (config *Config) appendTMMGUEST() { -+ config.qemuParams = append(config.qemuParams, "-object") -+ config.qemuParams = append(config.qemuParams, "tmm-guest,id=tmm0,sve-vector-length=128,num-pmu-counters=1") -+} -+ - // LaunchQemu can be used to launch a new qemu instance. - // - // The Config parameter contains a set of qemu parameters and settings. -@@ -3167,10 +3153,6 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) - config.appendQMPSockets() - config.appendMemory() - config.appendDevices(logger) -- config.appendRTC() -- config.appendGlobalParam() -- config.appendPFlashParam() -- config.appendVGA() - config.appendKnobs() - config.appendKernel() - config.appendBios() -@@ -3179,6 +3161,8 @@ func LaunchQemu(config Config, logger QMPLog) (*exec.Cmd, io.ReadCloser, error) - config.appendPidFile() - config.appendFwCfg(logger) - config.appendSeccompSandbox() -+ // append TMM GUEST -+ config.appendTMMGUEST() - - if err := config.appendCPUs(); err != nil { - return nil, nil, err -diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go -index bcc7d8ce7..07238aa03 100644 ---- a/src/runtime/pkg/katautils/create.go -+++ b/src/runtime/pkg/katautils/create.go -@@ -38,11 +38,7 @@ var systemdKernelParam = []vc.Param{ - }, - { - Key: "systemd.mask", -- Value: "systemd-networkd.service", -- }, -- { -- Key: "systemd.mask", -- Value: "systemd-networkd.socket", -+ Value: "NetworkManager.service", - }, - } - -diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go -index c61b44f67..5c726515d 100644 ---- a/src/runtime/virtcontainers/hypervisor.go -+++ b/src/runtime/virtcontainers/hypervisor.go -@@ -99,7 +99,7 @@ type RootfsDriver string - - const ( - // VirtioBlk is the Virtio-Blk rootfs driver. -- VirtioBlk RootfsDriver = "/dev/vda1" -+ VirtioBlk RootfsDriver = "/dev/vda" - - // Nvdimm is the Nvdimm rootfs driver. - Nvdimm RootfsType = "/dev/pmem0p1" -diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go -index 27d75ecc4..660a78dc5 100644 ---- a/src/runtime/virtcontainers/qemu.go -+++ b/src/runtime/virtcontainers/qemu.go -@@ -406,7 +406,6 @@ func (q *qemu) buildDevices(ctx context.Context, initrdPath string) ([]govmmQemu - - // Add bridges before any other devices. This way we make sure that - // bridge gets the first available PCI address i.e bridgePCIStartAddr -- devices = q.arch.appendBridges(devices) - - devices, err = q.arch.appendConsole(ctx, devices, console) - if err != nil { -diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go -index 9e05c5452..814a27d51 100644 ---- a/src/runtime/virtcontainers/qemu_arm64.go -+++ b/src/runtime/virtcontainers/qemu_arm64.go -@@ -29,10 +29,18 @@ const defaultQemuMachineType = QemuVirt - - const qmpMigrationWaitTimeout = 10 * time.Second - --const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host" -+// add cvm qemu machine options -+const defaultQemuMachineOptions = "gic-version=3,accel=kvm,kernel_irqchip=on" - - var kernelParams = []Param{ - {"iommu.passthrough", "0"}, -+ {"swiotlb", "262144,force"}, -+ {"console", "tty0"}, -+ {"console", "ttyAMA0"}, -+ {"kaslr.disabled", "1"}, -+ {"rodata", "off"}, -+ {"cma", "64M"}, -+ {"cvm_guest", "1"}, - } - - var supportedQemuMachine = govmmQemu.Machine{ --- -2.33.0 - diff --git a/confidential_container/0002-support-virtiofs.patch b/confidential_container/0002-support-virtiofs.patch deleted file mode 100644 index 997b198..0000000 --- a/confidential_container/0002-support-virtiofs.patch +++ /dev/null @@ -1,60 +0,0 @@ -From 7a83166689f8e0cfc703d2dbca7baf4fd3753012 Mon Sep 17 00:00:00 2001 -From: XiaoFeng Ma -Date: Mon, 12 Aug 2024 17:30:35 +0800 -Subject: [PATCH] support virtiofs - ---- - src/runtime/pkg/govmm/qemu/qemu.go | 12 ++++++++++++ - src/runtime/virtcontainers/virtiofsd.go | 9 +++++++-- - 2 files changed, 19 insertions(+), 2 deletions(-) - -diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go -index 1470a3533..812af4609 100644 ---- a/src/runtime/pkg/govmm/qemu/qemu.go -+++ b/src/runtime/pkg/govmm/qemu/qemu.go -@@ -1589,9 +1589,21 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string { - deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile)) - } - -+ if driver == "vhost-user-fs-pci" { -+ deviceParams = append(deviceParams, "iommu_platform=on") -+ } -+ - qemuParams = append(qemuParams, "-device") - qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) - -+ if driver == "vhost-user-fs-pci" { -+ qemuParams = append(qemuParams, "-object") -+ objParams := "memory-backend-file,id=mem,size=" + config.Memory.Size + ",mem-path=/dev/shm,share=on" -+ qemuParams = append(qemuParams, objParams) -+ qemuParams = append(qemuParams, "-numa") -+ qemuParams = append(qemuParams, "node,memdev=mem") -+ } -+ - return qemuParams - } - -diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go -index 3e02756eb..ff46691bd 100644 ---- a/src/runtime/virtcontainers/virtiofsd.go -+++ b/src/runtime/virtcontainers/virtiofsd.go -@@ -186,9 +186,14 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { - // Send logs to syslog - "--syslog", - // cache mode for virtiofsd -- "--cache=" + v.cache, -+ "-o", -+ "cache=" + v.cache, - // shared directory tree -- "--shared-dir=" + v.sourcePath, -+ "-o", -+ "source=" + v.sourcePath, -+ // Announce sub-mount points to the guest -+ "-o", -+ "announce_submounts", - // fd number of vhost-user socket - fmt.Sprintf("--fd=%v", FdSocketNumber), - } --- -2.33.0 - diff --git a/confidential_container/0001-kata.patch b/confidential_container/kata-container.patch similarity index 100% rename from confidential_container/0001-kata.patch rename to confidential_container/kata-container.patch -- Gitee